Added transitive source build for ATS.

infrastructure/docker/build/Dockerfile-tsb

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG RHEL_VERSION=8
+FROM centos:${RHEL_VERSION} as tsb-build
+RUN yum clean all && yum -y install git
+COPY GO_VERSION /
+RUN curl -L https://dl.google.com/go/go$(cat /GO_VERSION).linux-amd64.tar.gz | tar -C /usr/local -xz && ln -s /usr/local/go/bin/go /usr/bin/go
+# Fetch tested version of tsb:
+# <2020-12-01T20:37:43+00:00!alficles@gmail.com>: Upgraded to go-yaml v3 and added comments to patches.
+RUN mkdir -p /go/src/github.com/comcast/tsb && cd /go/src/github.com/comcast/tsb && git init && git remote add origin https://github.com/comcast/tsb && git fetch --depth=1 origin 851be3e137b20ba5d2d1eefd07cd303dc5f53865 && git reset --hard FETCH_HEAD
+RUN cd /go/src/github.com/comcast/tsb && CGOENABLED=0 go build -o /usr/bin/tsb
+
+FROM centos:${RHEL_VERSION}
+MAINTAINER dev@trafficcontrol.apache.org
+
+RUN	yum -y install git sudo
+
+# The current configuration does not require docker-compose for tsb, so a dummy script is included instead of docker-compose.
+# If docker-compose is required, the docker socket will need to be mounted inside this container as well.
+#ADD	https://github.com/docker/compose/releases/download/1.26.2/docker-compose-Linux-x86_64 /usr/bin/docker-compose
+RUN	echo -e "#!/usr/bin/env bash\necho docker-compose is not supported in this container and the build command will not work.\nexit 1" > /usr/bin/docker-compose
+
+RUN	chmod +x /usr/bin/docker-compose
+COPY	--from=tsb-build /usr/bin/tsb /usr/bin/tsb
+RUN	git config --global user.email "dev@trafficcontrol.apache.org"
+RUN	git config --global user.name "Apache Traffic Control"
+CMD	["/usr/bin/tsb", "cd", "/opt/tsb-ats", "fetch", "prebuild"]

infrastructure/docker/build/docker-compose-opt.yml

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: '2'
+
+services:
+  ats:
+    build:
+      dockerfile: infrastructure/docker/build/Dockerfile-tsb
+      context: ../../..
+    volumes:
+      - ../../../traffic_server/tsb:/opt/tsb-ats:z

infrastructure/docker/build/docker-compose-opt.yml.ats

@@ -0,0 +1 @@
+../../../traffic_server/tsb/docker-compose.yml

pkg

@@ -16,6 +16,7 @@
 SELF="${BASH_SOURCE[0]}"
 cd "$( dirname "${BASH_SOURCE[0]}" )"
 COMPOSE_FILE=./infrastructure/docker/build/docker-compose.yml
+COMPOSE_FILE_OPT=./infrastructure/docker/build/docker-compose-opt.yml
 
 # Check for dependencies
 if ! which docker >/dev/null 2>&1; then
@@ -88,20 +89,27 @@ RUN_OPTIONS=(-e RHEL_VERSION)
 
 # Parse command line arguments
 verbose=0
-while getopts :?78bdlpqv opt; do
+debug=0
+quiet=0
+all=0
+build_images=0
+while getopts :?78abdf:lopqv opt; do
 	case $opt in
 		\?)
 			PROJECTS=`$SELF -l | sed "s/^/  - /"`
 			<<-HELP_TEXT cat
 			Usage: $SELF [options] [projects]
-			  -7    Build RPMs targeting CentOS 7 (default)
-			  -8    Build RPMs targeting CentOS 8
-			  -b    Build builder Docker images before building projects
-			  -d    Disable compiler optimizations for debugging.
-			  -l    List available projects.
-			  -p    Pull builder Docker images, do not build them (default)
-			  -q    Quiet mode. Supresses output. (default)
-			  -v    Verbose mode. Lists all build output.
+			  -7           Build RPMs targeting CentOS 7 (default)
+			  -8           Build RPMs targeting CentOS 8
+			  -a           Build all projects, including optional ones.
+			  -b           Build builder Docker images before building projects
+			  -d           Disable compiler optimizations for debugging.
+			  -f <file>    Use <file> as the docker-compose. Default: $COMPOSE_FILE
+			  -l           List available projects.
+			  -o           Build from the optional list. Same as -f "$COMPOSE_FILE_OPT"
+			  -p           Pull builder Docker images, do not build them (default)
+			  -q           Quiet mode. Supresses output. (default)
+			  -v           Verbose mode. Lists all build output.
 
 			If no projects are listed, all projects will be packaged.
 			Valid projects:
@@ -115,6 +123,9 @@ while getopts :?78bdlpqv opt; do
 		8)
 			RHEL_VERSION=8
 			;;
+		a)
+			all=1
+			;;
 		b)
 			build_images=1
 			;;
@@ -123,16 +134,24 @@ while getopts :?78bdlpqv opt; do
 			# If DEBUG_BUILD is true, then Golang binaries will remain unoptimized and
 			# RPM packaging will not strip binaries.
 			RUN_OPTIONS+=(-e 'DEBUG_BUILD=true');
+			debug=1
+			;;
+		f)
+			COMPOSE_FILE="$OPTARG"
 			;;
 		l)
 			"${COMPOSECMD[@]}" -f $COMPOSE_FILE config --services
 			exit $?
 			;;
+		o)
+			COMPOSE_FILE="$COMPOSE_FILE_OPT"
+			;;
 		p)
 			build_images=0
 			;;
 		q)
 			exec >/dev/null 2>&1
+			quiet=1
 			;;
 		v)
 			verbose=1
@@ -142,9 +161,9 @@ done
 
 shift $((OPTIND-1))
 
-# If no specific packages are listed, run them all.
-if (( ! "$#" )); then
-	set -- `$SELF -l`
+# If no specific packages are listed, or -a is used, run them all.
+if (( ! "$#" || "$all" == 1 )); then
+	set -- `$SELF -f "$COMPOSE_FILE" -l`
 fi
 
 # Build each project in turn.
@@ -156,12 +175,27 @@ while (( "$#" )); do
 		if (( "$verbose" == 0 )); then
 			exec >/dev/null 2>&1
 		fi
+
+		# Build the project
 		if [[ $build_images -eq 1 ]]; then
 			"${COMPOSECMD[@]}" -f $COMPOSE_FILE build $1 || exit 1
 		else
 			"${COMPOSECMD[@]}" -f $COMPOSE_FILE pull $1 || exit 1
 		fi
 		"${COMPOSECMD[@]}" -f $COMPOSE_FILE run "${RUN_OPTIONS[@]}" --rm $1 || exit 1
+
+		# Check for a chained compose file for this particular project.
+		# A chained compose file will be named exactly the same as main docker-compose, with .service added,
+		# where <service> is the name of the specific service to be chained. The file may be a symlink to another
+		# compose file, in which case the symlink will be followed before it is processed.
+		# A "dist" symlink will be temporarily created in that directory to facilitate the collation of artefacts.
+		if [ -e "$COMPOSE_FILE.$1" ] ; then
+			ln -sf "$(dirname -- "$(realpath -e -- "$SELF")")/dist" "$(dirname -- "$(realpath -e -- "$COMPOSE_FILE.$1")")/dist" || exit 1
+			$SELF -f $(realpath -e "$COMPOSE_FILE.$1") $([ "$verbose" == 0 ] || echo "-v") $([ "$quiet" == 0 ] || echo "-q") $([ "$debug" == 0 ] || echo "-d") $([ "$build" == 0 ] || echo "-b") $("${COMPOSECMD[@]}" -f $(realpath -e "$COMPOSE_FILE.$1") config --services)
+			chained_exit=$?
+			rm "$(dirname -- "$(realpath -e -- "$COMPOSE_FILE.$1")")/dist"
+			[ $chained_exit == 0 ] || exit $chained_exit
+		fi
 	) || {
 		# Don't totally bail out, but make note of the failures.
 		failure=1
@@ -170,7 +204,9 @@ while (( "$#" )); do
 	shift
 done
 
-if (( $failure )); then
+[ "$all" == 0 ] || $SELF -o $([ "$verbose" == 0 ] || echo "-v") $([ "$quiet" == 0 ] || echo "-q") $([ "$debug" == 0 ] || echo "-d") || failure=1
+
+if [[ ! -z $badproj ]]; then
 	echo "Failed to build$badproj."
 fi
 

traffic_server/tsb/.dependency_license

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Patch files, cannot bear comments
+cjose.pic.patch, Apache-2.0
+jansson.pic.patch, Apache-2.0
+
+# YAML files, but are manipulated with scripts that would strip comments.
+patches.yml, Apache-2.0
+repos.yml, Apache-2.0

traffic_server/tsb/.gitignore

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+dist
+src

traffic_server/tsb/Dockerfile

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Fou:qndation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+FROM centos:7
+
+RUN	yum clean all \
+	&& yum install -y deltarpm epel-release centos-release-scl-rh \
+	&& yum-config-manager --enable rhel-server-rhscl-7-rpms \
+	&& yum clean all \
+	&& yum install -y \
+		autoconf \
+		automake \
+		devtoolset-7 \
+		ed \
+		expat-devel \
+		flex \
+		gcc-c++ \
+		git \
+		glibc-devel \
+		hwloc \
+		hwloc-devel \
+		libcap-devel \
+		libcurl-devel \
+		libtool \
+		libuuid-devel \
+		lua-devel \
+		luajit-devel \
+		make \
+		man \
+		nano \
+		ncurses-devel \
+		nmap-ncat \
+		openssl \
+		openssl-devel \
+		pcre \
+		pcre-devel \
+		perl-Digest-SHA \
+		perl-ExtUtils-MakeMaker \
+		perl-URI \
+		pkgconfig \
+		python3 \
+		rpm-build \
+		sudo \
+		tcl-devel \
+		zlib \
+		zlib-devel \
+	&& yum clean all
+
+COPY	jansson.pic.patch /opt/src/
+COPY	cjose.pic.patch /opt/src/
+ADD	https://bootstrap.pypa.io/get-pip.py /
+RUN	python get-pip.py
+RUN	pip install --user Sphinx
+COPY	run.sh /
+COPY	trafficserver.spec /rpmbuilddir/SPECS/trafficserver.spec
+COPY	traffic_server_jemalloc /rpmbuilddir/SOURCES/traffic_server_jemalloc
+RUN	/usr/sbin/useradd -u 176 -r ats -s /sbin/nologin -d /
+CMD	set -o pipefail; scl enable devtoolset-7 ./run.sh 2>&1 | tee /rpmbuilddir/RPMS/x86_64/build.log

traffic_server/tsb/cjose.pic.patch

@@ -0,0 +1,10 @@
+diff --git a/src/Makefile.am b/src/Makefile.am
+index 659e1c0..b8da1c3 100644
+--- a/src/Makefile.am
++++ b/src/Makefile.am
+@@ -1,4 +1,4 @@
+-AM_CFLAGS =-std=gnu99 --pedantic -Wall -Werror -g -O2 -I$(top_builddir)/include
++AM_CFLAGS =-std=gnu99 --pedantic -Wall -Werror -g -O2 -I$(top_builddir)/include -fpic
+
+ lib_LTLIBRARIES=libcjose.la
+ libcjose_la_CPPFLAGS= -I$(topdir)/include

traffic_server/tsb/docker-compose.yml

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+version: "2"
+services:
+    ats_build:
+        build:
+            context: .
+            dockerfile: Dockerfile
+        image: ats_build
+        volumes:
+            - ./src/jansson:/opt/src/jansson
+            - ./src/cjose:/opt/src/cjose
+            - ./src/openssl:/opt/src/openssl
+            - ./src/ats:/rpmbuilddir/SOURCES/src
+            - ./src/trafficcontrol/traffic_server/plugins/astats_over_http:/opt/src/astats_over_http
+            - ./dist:/rpmbuilddir/RPMS/x86_64

traffic_server/tsb/jansson.pic.patch

@@ -0,0 +1,9 @@
+diff --git a/src/Makefile.am b/src/Makefile.am
+index 5516927..118ea7c 100644
+--- a/src/Makefile.am
++++ b/src/Makefile.am
+@@ -1,3 +1,4 @@
++AM_CFLAGS = -fpic
+ EXTRA_DIST = jansson.def
+
+ include_HEADERS = jansson.h

traffic_server/tsb/patches.yml

@@ -0,0 +1,15 @@
+ats:
+    - f5ac00f3606bd826c52552e59f067d3fe3fcda33 # <2018-09-11T17:18:30+00:00!jrushford@apache.org> For Nexthop, add the capability to mark down origin servers utilizing traffic_ctl.
+    - 721f8ffca6ee8782573084c671e45ce392d239ef # <2018-06-21T17:21:58+00:00!jrushford@apache.org> add a reason tag to traffic_control host subcommand.
+    - d295784ba57af7e8e2dba09b85a1cbcaacf797f3 # <2019-05-16T19:33:17+00:00!jrushford@apache.org> Add an ignore_self_detect flag to parent.config so that the local cache host may be a member of a peering cache group.
+    - ef57a68f4f9d6f6dd5563cff82cddb8adcb8c28c # <2020-05-16T00:05:44+00:00!brian_olsen2@comcast.com> forward port of slice plugin from ats7
+    - abe7a933e403b88f1b13ab7f2ebdd6ce95202b42 # <2019-04-02T13:25:48+00:00!brian_olsen2@comcast.com> url_sig debug fix for when url is missing the signature query string
+    - 9ac557ba3c0f28979c996cb49a827940bfb569d8 # <2019-09-30T19:04:12+00:00!brian_olsen2@comcast.com> url_sig: fix memory leak with urlParse and pristine flag
+    - 72e582d3de630008641e7849f7cd42b11f6f081b # <2019-02-04T12:47:10-08:00!randallmeyer@yahoo.com> lua plugin: add support for relative path scripts; moves inline script to switch
+    - ad33e0fff0091093cd31fcfd429933299cb8bf5b # <2020-02-07T00:44:14-08:00!kichan@apache.org> Fix for lua plugin coredump problem during reload
+    - 1f13f346ef9ae4c0035f5755af4fd28b6f84f1e0 # <2020-04-20T14:59:44-06:00!brian_olsen2@comcast.com> code and documentation for lua states configuration and stats printing (#6571)
+    - fc66eea820c8ba44196f2051aa41aedd7b874b0b # <2020-03-16T19:30:54+00:00!jrushford@apache.org> Fixes a bug where getHostStatus() will create a host status record when none is found. A Host status record should only be created when a host is marked up or down when traffic_ctl is used to mark a host up or down.
+    - 5b38fb26737b38267615102775d81136a843075c # <2020-07-01T13:59:15+00:00!brian_olsen2@comcast.com> CacheRead: clear dir entry if doc is found to be truncated
+openssl:
+    - ec6788fb8ca0704b503c3a0030a142d9805895a6 # <2018-09-18T17:45:39+01:00!matt@openssl.org> Delay setting the sig algs until after the cert_cb has been called
+trafficcontrol: []