Merge pull request #121 from hnakamur/update_doc_ja_with_master

Update doc-ja branch with upstream/master

.gitignore

@@ -56,11 +56,14 @@ cmd/traffic_layout/traffic_layout
 cmd/traffic_via/traffic_via
 cmd/traffic_wccp/traffic_wccp
 
+lib/stamp-h1
+lib/ink_autoconf.h
+lib/ink_autoconf.h.in
+lib/ts/stamp-h1
 lib/ts/ink_autoconf.h
 lib/ts/ink_autoconf.h.in
 lib/ts/ink_config.h
 lib/ts/apidefs.h
-lib/ts/stamp-h1
 lib/ts/CompileParseRules
 lib/ts/ParseRulesCType
 lib/ts/ParseRulesCTypeToLower
@@ -83,6 +86,9 @@ iocore/eventsystem/test_Buffer
 iocore/eventsystem/test_Event
 proxy/config/records.config.default
 proxy/config/storage.config.default
+proxy/http2/test_Huffmancode
+
+plugins/header_rewrite/header_rewrite_test
 plugins/experimental/esi/*_test
 plugins/experimental/spdy/zstream_test
 plugins/experimental/sslheaders/test_sslheaders

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "lib/luajit"]
-	path = lib/luajit
-	url = http://luajit.org/git/luajit-2.0.git

.pr

@@ -0,0 +1,5 @@
+# Dummy file to close Github pull requests
+TS-4053: 9caf1d5c487120c2140a3127a44f494309700d6f
+TS-4057: 59cb070654a6e6a7acdd6509a770478d413f79a6
+fc59a5b9e6512397c3c5cedb3bd7436d7a8a65d1
+bcec70dc84a9a3eefc9a037c5a43e3422d4fc29a

Makefile.am

@@ -68,7 +68,7 @@ asf-dist-rc: asf-distdir
 asf-distdir:
 	@$(am__remove_distdir)
 	test -d .git && git clone . $(distdir)
-	cd $(distdir) && git submodule update --init && autoreconf -i
+	cd $(distdir) && autoreconf -i
 	$(RM) -r -- $(distdir)/autom4te.cache $(distdir)/ci `find $(distdir) -name .git -o -name .gitignore -o -name .gitmodules`
 
 asf-dist-sign: asf-dist
@@ -99,12 +99,45 @@ install-data-hook:
 rat:
 	java -jar $(top_srcdir)/ci/apache-rat-0.11-SNAPSHOT.jar -E $(top_srcdir)/ci/rat-regex.txt  -d $(top_srcdir)
 
-submodules:
-	git submodule update --init
+#
+# These are rules to make clang-format easy and fast to run. Run it with e.g.
+#    make -j clang-format
+#
+# If you make changes to directory structures, you must update this as well.
+#
+.PHONY:	clang-format-cmd clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \
+	clang-format-plugins clang-format-proxy clang-format-tools
+
+clang-format: 	clang-format-cmd clang-format-example clang-format-iocore clang-format-lib clang-format-mgmt \
+		clang-format-plugins clang-format-proxy clang-format-tools
+
+clang-format-cmd:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/cmd
+
+clang-format-example:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/example
+
+clang-format-iocore:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/iocore
+
+clang-format-lib:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/atscppapi
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/raft
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/records
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/ts
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/lib/wccp
+
+clang-format-mgmt:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/mgmt
+
+clang-format-plugins:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/plugins
+
+clang-format-proxy:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/proxy
 
-clang-format:
-	for f in `find $(top_srcdir) -iname \*.[ch] -o -iname \*.cc | fgrep -v -e lib/luajit -e lib/tsconfig`; do \
-	  echo $$f; clang-format -i $$f; done
+clang-format-tools:
+	@$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/tools
 
 help:
 	@echo 'all             default target for building the package' && \
@@ -119,9 +152,8 @@ help:
 	echo 'rel-candidate    recreate a signed relelease candidate source package and a signed git tag' && \
 	echo 'distcheck        verify dist by performing VPATH build and then distclean' && \
 	echo 'rat              produce a RAT licence compliance report of the source' && \
-	echo 'submodules       update the git submodules' && \
 	echo 'doxygen          generate doxygen docs in doc/html dir' && \
-	echo 'clang-format     run clang-format over most C and C++ files (not git submodules/trees)' && \
+	echo 'clang-format     run clang-format over most C and C++ files (not git subtrees)' && \
 	echo 'help             display this list of make targets' && \
 	echo 'install          install by copying the built files to system-wide dirs' && \
 	echo 'install-strip    same as install but then strips debugging symbols' && \

NOTICE

@@ -75,6 +75,9 @@ Copyright (C) 2012 Google Inc.
 collapsed_collapsing: Plugin for connection collapsing to origin
 Copyright (C) 2014 Yahoo! Inc.  All rights reserved.
 
+multiplexer: Plugin for request multiplixing
+Copyright (C) 2015 Yahoo! Inc.  All rights reserved.
+
 ~~~
 
 healthchecks: Plugin for ATS healthchecks.
@@ -91,3 +94,13 @@ lib/ts/HashSip.cc contains code derived from code at https://github.com/floodybe
 The code is public domain.
 
 The algorithm info can be found at: https://131002.net/siphash/
+
+~~
+
+lib/raft developed by Google.
+Copyright 2014 Google Inc. All Rights Reserved.
+
+~~
+
+plugins/experimental/memcache/protocol_binary.h developed by Sun Microsystems, Inc.
+Copyright (c) <2008>, Sun Microsystems, Inc.  All rights reserved.

README

@@ -45,7 +45,9 @@ plugins to build large scale web applications.
       |-- congest/ ....... Congestion control
       |-- hdrs/ .......... Headers parsing and management
       |-- http/ .......... The actual HTTP protocol implementation
+      |---http2/ ......... HTTP/2 implementation
       |-- logging/ ....... Flexible logging
+      |-- spdy/ .......... SPDY implementation
   |-- README ............. intro, links, build info
   |-- README-EC2 ......... info on EC2 support
   |-- REVIEWERS .......... (incomplete) list of areas with committer interest

build/ax_check_openssl.m4

@@ -83,12 +83,12 @@ AC_DEFUN([AX_CHECK_OPENSSL], [
                 if test -d "$ssldir/lib64"; then
                   OPENSSL_LDFLAGS="-L$ssldir/lib64"
                   if test "$overriden_with_ssl"; then
-                    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-rpath ${ssldir}/lib64])
+                    TS_ADDTO_RPATH(${ssldir}/lib64)
                   fi
                 else
                   OPENSSL_LDFLAGS="-L$ssldir/lib"
                   if test "$overriden_with_ssl"; then
-                    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-rpath ${ssldir}/lib])
+                    TS_ADDTO_RPATH(${ssldir}/lib)
                   fi
                 fi
                 OPENSSL_LIBS="-lssl -lcrypto"

build/common.m4

@@ -136,6 +136,16 @@ AC_DEFUN([TS_ADDTO], [
   fi
 ])dnl
 
+dnl
+dnl TS_ADDTO_RPATH(path)
+dnl
+dnl   Adds path to variable with the '-rpath' directive.
+dnl
+AC_DEFUN([TS_ADDTO_RPATH], [
+  AC_MSG_NOTICE([adding $1 to RPATH])
+  TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R$1])
+])dnl
+
 dnl
 dnl TS_REMOVEFROM(variable, value)
 dnl

build/crypto.m4

@@ -134,7 +134,7 @@ AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [
   enable_cert_cb=yes
 
   TS_ADDTO(LIBS, [$OPENSSL_LIBS])
-
+  AC_MSG_CHECKING([for SSL_CTX_set_cert_cb])
   AC_LINK_IFELSE(
   [
     AC_LANG_PROGRAM([[
@@ -157,7 +157,7 @@ AC_DEFUN([TS_CHECK_CRYPTO_CERT_CB], [
 
   LIBS=$_cert_saved_LIBS
 
-  AC_MSG_CHECKING(whether to enable Certificate callback support)
+  AC_MSG_CHECKING(whether to enable TLS certificate callback support)
   AC_MSG_RESULT([$enable_cert_cb])
   TS_ARG_ENABLE_VAR([use], [cert-cb])
   AC_SUBST(use_cert_cb)

build/jemalloc.m4

@@ -56,7 +56,7 @@ if test "$enable_jemalloc" != "no"; then
   if test "$jemalloc_base_dir" != "/usr"; then
     TS_ADDTO(CPPFLAGS, [-I${jemalloc_include}])
     TS_ADDTO(LDFLAGS, [-L${jemalloc_ldflags}])
-    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R${jemalloc_ldflags}])
+    TS_ADDTO_RPATH(${jemalloc_ldflags})
   fi
   # On Darwin, jemalloc symbols are prefixed with je_. Search for that first, then fall back
   # to unadorned symbols.

build/lzma.m4

@@ -82,7 +82,7 @@ if test "$enable_lzma" != "no"; then
   if test "$lzma_base_dir" != "/usr"; then
     TS_ADDTO(CPPFLAGS, [-I${lzma_include}])
     TS_ADDTO(LDFLAGS, [-L${lzma_ldflags}])
-    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R${lzma_ldflags}])
+    TS_ADDTO_RPATH(${lzma_ldflags})
   fi
   AC_SEARCH_LIBS([lzma_code], [lzma], [lzma_have_libs=1])
   if test "$lzma_have_libs" != "0"; then

build/pcre.m4

@@ -93,7 +93,7 @@ if test "$enable_pcre" != "no"; then
   if test "$pcre_base_dir" != "/usr"; then
     TS_ADDTO(CPPFLAGS, [-I${pcre_include}])
     TS_ADDTO(LDFLAGS, [-L${pcre_ldflags}])
-    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R${pcre_ldflags}])
+    TS_ADDTO_RPATH(${pcre_ldflags})
   fi
   AC_SEARCH_LIBS([pcre_exec], [pcre], [pcre_have_libs=1])
   if test "$pcre_have_libs" != "0"; then

build/plugins.mk

@@ -25,8 +25,6 @@ TS_PLUGIN_LDFLAGS = \
 TS_PLUGIN_CPPFLAGS = \
   -I$(top_builddir)/proxy/api \
   -I$(top_srcdir)/proxy/api \
-  -I$(top_builddir)/lib/ts \
-  -I$(top_srcdir)/lib/ts \
   -I$(top_srcdir)/lib
 
 # Provide a default AM_CPPFLAGS. Automake handles this correctly, but libtool

build/tcmalloc.m4

@@ -41,7 +41,7 @@ AC_ARG_WITH([tcmalloc], [AC_HELP_STRING([--with-tcmalloc=DIR], [use the tcmalloc
     if test "x$withval" != "xyes" && test "x$withval" != "x"; then
       tcmalloc_ldflags="$withval/lib"
       TS_ADDTO(LDFLAGS, [-L${tcmalloc_ldflags}])
-      TS_ADDTO(LIBTOOL_LINK_FLAGS, [-rpath ${tcmalloc_ldflags}])
+      TS_ADDTO_RPATH(${tcmalloc_ldflags})
     fi
     AC_SEARCH_LIBS([tc_cfree], ${with_tcmalloc_lib}, [tcmalloc_have_lib=1])
     if test "$tcmalloc_have_lib" != "0"; then

build/xml.m4

@@ -92,7 +92,7 @@ AC_DEFUN([TS_CHECK_XML_LIBXML2], [
       fi
       if test -d "$libxml2_ldflags" ; then
         TS_ADDTO(LDFLAGS, [-L${libxml2_ldflags}])
-        TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R${libxml2_ldflags}])
+        TS_ADDTO_RPATH(${libxml2_ldflags})
       fi
       TS_ADDTO(LIBS, -lxml2)
       enable_xml=yes
@@ -179,7 +179,7 @@ if test "$enable_expat" != "no"; then
   if test "$expat_base_dir" != "/usr"; then
     TS_ADDTO(CPPFLAGS, [-I${expat_include}])
     TS_ADDTO(LDFLAGS, [-L${expat_ldflags}])
-    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-R${expat_ldflags}])
+    TS_ADDTO_RPATH(${expat_ldflags})
   fi
   AC_SEARCH_LIBS([XML_SetUserData], [expat], [expat_have_libs=1])
   if test "$expat_have_libs" != "0"; then

build/zlib.m4

@@ -82,7 +82,7 @@ if test "$enable_zlib" != "no"; then
   if test "$zlib_base_dir" != "/usr"; then
     TS_ADDTO(CPPFLAGS, [-I${zlib_include}])
     TS_ADDTO(LDFLAGS, [-L${zlib_ldflags}])
-    TS_ADDTO(LIBTOOL_LINK_FLAGS, [-rpath ${zlib_ldflags}])
+    TS_ADDTO_RPATH(${zlib_ldflags})
   fi
   AC_SEARCH_LIBS([compressBound], [z], [zlib_have_libs=1])
   if test "$zlib_have_libs" != "0"; then

ci/jenkins/bin/in_tree.sh

@@ -32,8 +32,7 @@ autoreconf -fi
     --enable-example-plugins \
     --enable-test-tools \
     --enable-wccp \
-    ${enable_cppapi} \
-    CORES=2
+    ${enable_cppapi}
 
 ${ATS_MAKE} -j5 V=1
 #${ATS_MAKE} check VERBOSE=Y

ci/jenkins/bin/out_of_tree.sh

@@ -18,6 +18,10 @@
 
 # This does intentionally not run the regressions, it's primarily a "build" test
 
+# Test if we should enable CPPAPI (only 5.0 and later for now)
+enable_cppapi="--enable-cppapi"
+test "${JOB_NAME#*-6.0.x}" == "${JOB_NAME}" && enable_cppapi=""
+
 cd "${WORKSPACE}/src"
 
 autoreconf -fi
@@ -29,7 +33,7 @@ mkdir -p BUILDS && cd BUILDS
     --enable-example-plugins \
     --enable-test-tools \
     --enable-wccp \
-    CORES=2
+    ${enable_cppapi}
 
 ${ATS_MAKE} -j5 V=1
 #${ATS_MAKE} check VERBOSE=Y

ci/regression

@@ -89,6 +89,7 @@ configure() {
       --enable-example-plugins \
       --enable-test-tools \
       --enable-experimental-plugins \
+      --enable-cppapi \
       $(extras) \
       CC="$CC" \
       CXX="$CXX" \
@@ -101,9 +102,10 @@ build() {
   ( cd $OBJROOT && $MAKE install )
 }
 
+# Run extended (level 3) regressions.
 regress() {
   ( cd $OBJROOT && $MAKE check ) && \
-  $DSTROOT/bin/traffic_server -R 1
+  $DSTROOT/bin/traffic_server -R 3
 }
 
 CC=${CC:-$(prog cc)}

ci/tsqa/requirements.txt

@@ -4,3 +4,5 @@
 https://github.com/apache/trafficserver-qa/archive/master.zip
 pyyaml
 pyOpenSSL
+# TODO: can't do python_version in requirements.txt files
+#hyper; python_version >= '2.7'

ci/tsqa/tests/test_body_factory.py

@@ -0,0 +1,73 @@
+'''
+Test body_factory
+'''
+
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import requests
+import logging
+import random
+import tsqa.test_cases
+import helpers
+
+log = logging.getLogger(__name__)
+
+
+class TestDomainSpecificBodyFactory(helpers.EnvironmentCase):
+    '''
+    Tests for how body factory works with requests of different domains
+    '''
+    @classmethod
+    def setUpEnv(cls, env):
+        cls.configs['records.config']['CONFIG'].update({
+            'proxy.config.body_factory.enable_customizations': 3,  # enable domain specific body factory
+        })
+        cls.configs['remap.config'].add_line(
+            'map / http://www.linkedin.com/ @action=deny'
+        )
+        cls.body_factory_dir = os.path.join(cls.environment.layout.prefix, cls.configs['records.config']['CONFIG']['proxy.config.body_factory.template_sets_dir'])
+        cls.domain_directory = ['www.linkedin.com', '127.0.0.1', 'www.foobar.net']
+        for directory_item in cls.domain_directory:
+            current_dir = os.path.join(cls.body_factory_dir, directory_item)
+            try:
+                os.mkdir(current_dir)
+            except:
+                pass
+            fname = os.path.join(current_dir, "access#denied")
+            with open(fname, "w") as f:
+              f.write(directory_item)
+            fname = os.path.join(current_dir, ".body_factory_info")
+            with open(fname, "w") as f:
+              pass
+
+    def test_domain_specific_body_factory(self):
+      times = 1000
+      no_dir_domain = 'www.nodir.com'
+      self.domain_directory.append(no_dir_domain)
+      self.assertEqual(4, len(self.domain_directory))
+      url = 'http://127.1.0.1:{0}'.format(self.configs['records.config']['CONFIG']['proxy.config.http.server_ports'])
+      for i in xrange(times):
+          domain = random.choice(self.domain_directory)
+          headers = {'Host': domain}
+          r = requests.get(url, headers=headers)
+          domain_in_response = no_dir_domain
+          for domain_item in self.domain_directory:
+              if domain_item in r.text:
+                   domain_in_response = domain_item
+                   break
+          self.assertEqual(domain, domain_in_response)