Merge branch 'master' into quic-latest

* master:
  Backing out my update of our jenkin's autest file. (#7118)
  Don't send image/webp responses from cache to broswers that don't support it (#7104)
  Updating our autest suite to require Python3.6 (#7113)
  Squashed commit of the following: (#7110)
  Supporting out of source builds for AuTests. (#7109)
  Fixes uninitialized variables found by Xcode (#7100)
  Add cross references between server session sharing match and upstream connection tracking match. (#7038)

doc/admin-guide/files/records.config.en.rst

@@ -975,9 +975,15 @@ mptcp
 
 .. note::
 
-   Server sessions to different ports never match even if the FQDN and IP
+   Server sessions to different upstream ports never match even if the FQDN and IP
    address match.
 
+.. note::
+
+   :ts:cv:`Upstream session tracking <proxy.config.http.per_server.connection.max>` uses a similar
+   set of options for matching sessions, but is :ts:cv:`set independently
+   <proxy.config.http.per_server.connection.match>` from session sharing.
+
 .. ts:cv:: CONFIG proxy.config.http.server_session_sharing.pool STRING thread
 
    Control the scope of server session re-use if it is enabled by
@@ -1504,6 +1510,11 @@ Origin Server Connect Attempts
 
    To disable upstream server grouping, set :ts:cv:`proxy.config.http.per_server.connection.max` to ``0``.
 
+.. note::
+
+   This setting is independent of the :ts:cv:`setting for upstream session sharing matching
+   <proxy.config.http.server_session_sharing.match>`.
+
 .. ts:cv:: CONFIG proxy.config.http.per_server.connection.queue_size INT 0
    :reloadable:
 

doc/admin-guide/plugins/icap.en.rst

@@ -1,4 +1,5 @@
 .. _icap-plugin:
+.. include:: ../../common.defs
 
 ICAP Plugin
 ***********

doc/admin-guide/plugins/memory_profile.en.rst

@@ -1,3 +1,5 @@
+.. include:: ../../common.defs
+
 Memory_profile Plugin
 *********************
 

proxy/http/HttpBodyFactory.cc

@@ -372,8 +372,6 @@ HttpBodyFactory::fabricate(StrList *acpt_language_list, StrList *acpt_charset_li
   char *buffer;
   const char *pType = context->txn_conf->body_factory_template_base;
   const char *set;
-  HttpBodyTemplate *t = nullptr;
-  HttpBodySet *body_set;
   char template_base[PATH_NAME_MAX];
 
   if (set_return) {
@@ -409,6 +407,9 @@ HttpBodyFactory::fabricate(StrList *acpt_language_list, StrList *acpt_charset_li
   if (set_return) {
     *set_return = set;
   }
+
+  HttpBodyTemplate *t   = nullptr;
+  HttpBodySet *body_set = nullptr;
   if (pType != nullptr && 0 != *pType && 0 != strncmp(pType, "NONE", 4)) {
     sprintf(template_base, "%s_%s", pType, type);
     t = find_template(set, template_base, &body_set);

proxy/http/HttpTransactCache.cc

@@ -486,13 +486,6 @@ HttpTransactCache::calculate_quality_of_match(const OverridableHttpConfigParams
   @return quality (-1: no match, 0..1: poor..good).
 
 */
-static inline bool
-do_content_types_match(char *type1, char *subtype1, char *type2, char *subtype2)
-{
-  return ((is_asterisk(type1) || is_empty(type1) || (strcasecmp(type1, type2) == 0)) &&
-          (is_asterisk(subtype1) || is_empty(subtype1) || (strcasecmp(subtype1, subtype2) == 0)));
-}
-
 float
 HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MIMEField *content_field)
 {
@@ -526,6 +519,9 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI
   // Parse the type and subtype of the Content-Type field.
   HttpCompat::parse_mime_type(c_param->str, c_type, c_subtype, sizeof(c_type), sizeof(c_subtype));
 
+  // Special case for webp because Safari is has Accept: */*, but doesn't support webp
+  bool content_type_webp = ((strcasecmp("webp", c_subtype) == 0) && (strcasecmp("image", c_type) == 0));
+
   // Now loop over Accept field values.
   // TODO: Should we check the return value (count) from this?
   accept_field->value_get_comma_list(&a_values_list);
@@ -549,19 +545,25 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI
     char a_type[32], a_subtype[32];
     HttpCompat::parse_mime_type(a_param->str, a_type, a_subtype, sizeof(a_type), sizeof(a_subtype));
 
-    //      printf("matching Content-type; '%s/%s' with Accept value '%s/%s'\n",
-    //             c_type,c_subtype,a_type,a_subtype);
-
-    // Is there a wildcard in the type or subtype?
-    if (is_asterisk(a_type)) {
-      wildcard_type_present = true;
-      wildcard_type_q       = HttpCompat::find_Q_param_in_strlist(&a_param_list);
-    } else if (is_asterisk(a_subtype) && (strcasecmp(a_type, c_type) == 0)) {
-      wildcard_subtype_present = true;
-      wildcard_subtype_q       = HttpCompat::find_Q_param_in_strlist(&a_param_list);
-    } else {
-      // No wildcard. Do explicit matching of accept and content values.
-      if (do_content_types_match(a_type, a_subtype, c_type, c_subtype)) {
+    Debug("http_match", "matching Content-type; '%s/%s' with Accept value '%s/%s'\n", c_type, c_subtype, a_type, a_subtype);
+
+    bool wildcard_found = true;
+    // Only do wildcard checks if the content type is not image/webp
+    if (content_type_webp == false) {
+      // Is there a wildcard in the type or subtype?
+      if (is_asterisk(a_type)) {
+        wildcard_type_present = true;
+        wildcard_type_q       = HttpCompat::find_Q_param_in_strlist(&a_param_list);
+      } else if (is_asterisk(a_subtype) && (strcasecmp(a_type, c_type) == 0)) {
+        wildcard_subtype_present = true;
+        wildcard_subtype_q       = HttpCompat::find_Q_param_in_strlist(&a_param_list);
+      } else {
+        wildcard_found = false;
+      }
+    }
+    if (content_type_webp == true || wildcard_found == false) {
+      // No wildcard or the content type is image/webp. Do explicit matching of accept and content values.
+      if ((strcasecmp(a_type, c_type) == 0) && (strcasecmp(a_subtype, c_subtype) == 0)) {
         float tq;
         tq = HttpCompat::find_Q_param_in_strlist(&a_param_list);
         q  = (tq > q ? tq : q);
@@ -582,6 +584,7 @@ HttpTransactCache::calculate_quality_of_accept_match(MIMEField *accept_field, MI
   if ((q == -1.0) && (wildcard_type_present == true)) {
     q = wildcard_type_q;
   }
+
   return (q);
 }
 
@@ -1309,7 +1312,8 @@ HttpTransactCache::match_response_to_request_conditionals(HTTPHdr *request, HTTP
 
   // If-Match: must match strongly //
   if (request->presence(MIME_PRESENCE_IF_MATCH)) {
-    int raw_etags_len, comma_sep_tag_list_len;
+    int raw_etags_len              = 0;
+    int comma_sep_tag_list_len     = 0;
     const char *raw_etags          = response->value_get(MIME_FIELD_ETAG, MIME_LEN_ETAG, &raw_etags_len);
     const char *comma_sep_tag_list = nullptr;
 

tests/README.md

@@ -44,7 +44,7 @@ To run autest manually, the recommended way is to follow these steps:
 
 AuTest and the relevant tools can be install manually instead of using the wrapper script. By doing this, it is often easier to debug issues with the testing system, or the tests. There are two ways this can be done.
 1. Run the bootstrap script then source the path with a "source ./env-test/bin/activate" command. At this point autest command should run without the wrapper script
-2. Make sure you install python 3.5 or better on your system. From there install these python packages ( ie pip install ):
+2. Make sure you install python 3.6 or better on your system. From there install these python packages ( ie pip install ):
   - hyper
   - git+https://bitbucket.org/autestsuite/reusable-gold-testing-system.git
   - [traffic-replay](https://bitbucket.org/autestsuite/trafficreplay/src/master/) (This will automatically install [MicroDNS](https://bitbucket.org/autestsuite/microdns/src/master/), [MicroServer](https://bitbucket.org/autestsuite/microserver/src/master/), [TrafficReplayLibrary](https://bitbucket.org/autestsuite/trafficreplaylibrary/src/master/), and dnslib as part of the dependencies.)
@@ -53,7 +53,7 @@ AuTest and the relevant tools can be install manually instead of using the wrapp
 When writing for the AuTest system please refer to the current [Online Documentation](https://autestsuite.bitbucket.io/) for general use of the system. To use CurlHeader tester for testing output of curl, please refer to [CurlHeader README](gold_tests/autest-site/readme.md)
 
 ## Documentation of AuTest extension for ATS.
-Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.5 or better is used. There is also a new command line argumented added specifically for Trafficserver:
+Autest allows for the creation of extensions to help specialize and simplify test writing for a given application domain. Minus API addition the extension code will check that python 3.6 or better is used. There is also a new command line argumented added specifically for Trafficserver:
 
 --ats-bin < path to bin directory >
 

tests/gold_tests/autest-site/init.cli.ext

@@ -18,16 +18,24 @@
 
 import sys
 
-if sys.version_info < (3, 5, 0):
+if sys.version_info < (3, 6, 0):
     host.WriteError(
-        "You need python 3.5 or later to run these tests\n", show_stack=False)
+        "You need python 3.6 or later to run these tests\n", show_stack=False)
 
-autest_version = "1.7.2"
+autest_version = "1.8.1"
 if AuTestVersion() < autest_version:
     host.WriteError(
-        "Tests need AuTest version {ver} or better\n Please update AuTest:\n  pip install --upgrade autest\n".format(ver=autest_version), show_stack=False)
+        "Tests need AuTest version {needed_version} or better, found version {found_version}\n"
+            "Please update AuTest:\n  pip install --upgrade autest\n".format(
+            needed_version=autest_version,
+            found_version=AuTestVersion()),
+        show_stack=False)
 
 
 Settings.path_argument(["--ats-bin"],
                        required=True,
                        help="A user provided directory to ATS bin")
+
+Settings.path_argument(["--build-root"],
+                       required=False,
+                       help="The location of the build root for out of source builds")

tests/gold_tests/autest-site/setup.cli.ext

@@ -17,13 +17,23 @@
 #  limitations under the License.
 
 import json
-import subprocess
+import os
+from os.path import dirname
 import pprint
+import subprocess
 
 if Arguments.ats_bin is not None:
     # Add environment variables
     ENV['ATS_BIN'] = Arguments.ats_bin
 
+if Arguments.build_root is not None:
+    ENV['BUILD_ROOT'] = Arguments.build_root
+else:
+    # Assume the build root is the same directory tree as the test location.
+    ENV['BUILD_ROOT'] = dirname(dirname(dirname(AutestSitePath)))
+
+host.WriteVerbose(['ats'], "Test build root: {}:".format(ENV['BUILD_ROOT']))
+
 if ENV['ATS_BIN'] is not None:
     # Add variables for Tests
     traffic_layout = os.path.join(ENV['ATS_BIN'], "traffic_layout")
@@ -78,9 +88,12 @@ if ENV['ATS_BIN'] is not None:
                 host.WriteError("tsxs is broken. Aborting tests", show_stack=False)
     host.WriteVerbose(['ats'], "Traffic server build flags:\n", pprint.pformat(out))
     Variables.update(out)
-Variables.AtsExampleDir = os.path.join(AutestSitePath, '../../../example')
-Variables.AtsTestToolsDir = os.path.join(AutestSitePath, '../../tools')
-Variables.AtsTestPluginsDir = os.path.join(AutestSitePath, '../../tools/plugins/.libs')
+
+Variables.AtsExampleDir = os.path.join(AutestSitePath, '..', '..', '..', 'example')
+Variables.AtsTestToolsDir = os.path.join(AutestSitePath, '..', '..', 'tools')
+Variables.BuildRoot = ENV['BUILD_ROOT']
+Variables.AtsTestPluginsDir = os.path.join(Variables.BuildRoot, 'tests', 'tools', 'plugins', '.libs')
+Variables.AtsBuildGoldTestsDir = os.path.join(Variables.BuildRoot, 'tests', 'gold_tests')
 
 # modify delay times as we always have to kill Trafficserver
 # no need to wait

tests/gold_tests/chunked_encoding/chunked_encoding.test.py

@@ -16,6 +16,8 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import os
+
 Test.Summary = '''
 Test chunked encoding processing
 '''
@@ -94,7 +96,7 @@
 
 # smuggle-client is built via `make`. Here we copy the built binary down to the
 # test directory so that the test runs in this file can use it.
-Test.Setup.Copy('smuggle-client')
+Test.Setup.Copy(os.path.join(Test.Variables.AtsBuildGoldTestsDir, 'chunked_encoding', 'smuggle-client'))
 
 # HTTP1.1 GET: www.example.com
 tr = Test.AddTestRun()

tests/gold_tests/continuations/session_id.test.py

@@ -39,7 +39,8 @@
 ts.addSSLfile("ssl/server.pem")
 ts.addSSLfile("ssl/server.key")
 
-Test.PrepareTestPlugin(os.path.join(Test.TestDirectory, 'plugins', '.libs', 'session_id_verify.so'), ts)
+Test.PrepareTestPlugin(os.path.join(Test.Variables.AtsBuildGoldTestsDir,
+                                    'continuations', 'plugins', '.libs', 'session_id_verify.so'), ts)
 
 ts.Disk.records_config.update({
     'proxy.config.diags.debug.enabled': 1,

tests/gold_tests/headers/accept_webp.test.py

@@ -0,0 +1,79 @@
+'''
+Test how we handle image/webp
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+Test.Summary = '''
+Checking that we don't serve image/webp to clients that do not support it
+'''
+
+Test.ContinueOnFail = True
+
+# Define default ATS
+ts = Test.MakeATSProcess("ts")
+server = Test.MakeOriginServer("server")
+
+testName = "accept_webp"
+request_header = {
+    "headers": "GET / HTTP/1.1\r\nHost: www.example.com\r\nAccept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5\r\n\r\n",
+    "timestamp": "1469733493.993",
+    "body": ""}
+response_header = {
+    "headers": "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Type: image/webp\r\nCache-Control: max-age=300\r\n",
+    "timestamp": "1469733493.993",
+    "body": "xxx"}
+server.addResponse("sessionlog.json", request_header, response_header)
+
+# ATS Configuration
+ts.Disk.records_config.update({
+    'proxy.config.diags.debug.enabled': 1,
+    'proxy.config.diags.debug.tags': 'http_match',
+    'proxy.config.http.cache.ignore_accept_mismatch': 0,
+    'proxy.config.http.insert_response_via_str': 3,
+    'proxy.config.http.cache.http': 1,
+    'proxy.config.http.wait_for_cache': 1,
+})
+
+ts.Disk.remap_config.AddLine(
+    'map http://www.example.com http://127.0.0.1:{0}'.format(server.Variables.Port)
+)
+
+# Test 1 - Request with image/webp support from the origin
+tr = Test.AddTestRun()
+tr.Processes.Default.StartBefore(server, ready=When.PortOpen(server.Variables.Port))
+tr.Processes.Default.StartBefore(Test.Processes.ts)
+tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format(
+    ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.stderr = "gold/accept_webp.gold"
+tr.StillRunningAfter = ts
+
+# Test 2 - Request with image/webp support from cache
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format(
+    ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.stderr = "gold/accept_webp_cache.gold"
+tr.StillRunningAfter = ts
+
+# Test 3 - Request without image/webp support going to the origin - NOTE: the origin can't change the content-type :(
+tr = Test.AddTestRun()
+tr.Processes.Default.Command = 'curl -s -D - -v --ipv4 --http1.1 -H "Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5" -H "Host: www.example.com" http://localhost:{0}/'.format(
+    ts.Variables.port)
+tr.Processes.Default.ReturnCode = 0
+tr.Processes.Default.Streams.stderr = "gold/accept_webp_jpeg.gold"
+tr.StillRunningAfter = ts

tests/gold_tests/headers/gold/accept_webp.gold

@@ -0,0 +1,16 @@
+``
+> GET /``
+> Host: www.example.com``
+> User-Agent: curl/``
+> Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5
+``
+< HTTP/1.1 200 OK
+< Content-Type: image/webp
+< Cache-Control: max-age=300
+< Content-Length: 3
+< Date: ``
+< Age: ``
+< Connection: keep-alive
+< Via: http/1.1 `` (ApacheTrafficServer/`` [uScMsSfWpSeN:t cCMp sS])
+< Server: ATS/``
+``

tests/gold_tests/headers/gold/accept_webp_cache.gold

@@ -0,0 +1,16 @@
+``
+> GET /``
+> Host: www.example.com``
+> User-Agent: curl/``
+> Accept: image/webp,image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5
+``
+< HTTP/1.1 200 OK
+< Content-Type: image/webp
+< Cache-Control: max-age=300
+< Content-Length: 3
+< Date: ``
+< Age: ``
+< Connection: keep-alive
+< Via: http/1.1 `` (ApacheTrafficServer/`` [uScHs f p eN:t cCHp s ])
+< Server: ATS/``
+``

tests/gold_tests/headers/gold/accept_webp_jpeg.gold

@@ -0,0 +1,16 @@
+``
+> GET /``
+> Host: www.example.com``
+> User-Agent: curl/``
+> Accept: image/png,image/svg+xml,image/*;q=0.8,video/*;q=0.8,*/*;q=0.5
+``
+< HTTP/1.1 200 OK
+< Content-Type: image/webp
+< Cache-Control: max-age=300
+< Content-Length: 3
+< Date: ``
+< Age: ``
+< Connection: keep-alive
+< Via: http/1.1 `` (ApacheTrafficServer/`` [uScMsSfWpSeN:t cCMp sS])
+< Server: ATS/``
+``