Backport changes needed to have linux-pkg run on 6.0/migration

lib/common.sh

@@ -89,7 +89,7 @@ function die() {
 }
 
 function logmust() {
-	echo Running: "$@"
+	[[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2
 	"$@" || die "failed command '$*'"
 }
 
@@ -470,7 +470,7 @@ function install_pkgs() {
 		echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \
 			"apt-get install -y $*"
 		sudo env DEBIAN_FRONTEND=noninteractive apt-get install \
-			-y "$@" && return
+			-y --allow-downgrades "$@" && return
 		echo "apt-get install failed, retrying."
 		sleep 10
 	done
@@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() {
 	url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}"
 
 	logmust cd "$WORKDIR/artifacts"
-	logmust wget -nv "$url"
+	fetch_file_from_artifactory "$url"
+}
+
+function fetch_file_from_artifactory() {
+	local url="$1"
+	local sha256_expected="$2"
+	local file sha256_actual
+
+	[[ -n "$url" ]] || die "url argument is missing."
+	file="$(basename "$url")"
+
+	logmust wget -nv "$url" -O "$file"
+
+	if [[ -n "$sha256_expected" ]]; then
+		sha256_actual="$(sha256sum "$file" | awk '{print $1}')"
+		if [[ "$sha256_expected" != "$sha256_actual" ]]; then
+			die "SHA256 mismatch. Expected: $sha256_expected," \
+				"Actual: $sha256_actual"
+		fi
+	fi
 }
 
 #

package-lists/build/main.pkgs

@@ -18,6 +18,8 @@ grub2
 libkdumpfile
 make-jpkg
 makedumpfile
+masking
+misc-debs
 nfs-utils
 performance-diagnostics
 ptools
@@ -26,4 +28,4 @@ recovery-environment
 savedump
 sdb
 targetcli-fb
-td-agent-prebuilt
+virtualization

packages/adoptopenjdk/config.sh

@@ -19,8 +19,9 @@
 DEFAULT_PACKAGE_GIT_URL=none
 PACKAGE_DEPENDENCIES="make-jpkg"
 
-tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
-jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
+_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz"
+_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1"
+_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64"
 
 function prepare() {
 	logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb
@@ -29,15 +30,15 @@ function prepare() {
 function fetch() {
 	logmust cd "$WORKDIR/"
 
-	local url="http://artifactory.delphix.com/artifactory"
+	local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile"
 
-	logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile"
+	logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256"
 }
 
 function build() {
 	logmust cd "$WORKDIR/"
 
-	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<<y
+	logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$_tarfile" <<<y
 
 	logmust mv ./*deb "$WORKDIR/artifacts/"
 	#
@@ -49,7 +50,7 @@ function build() {
 	# packages, such as the app-gate, decide to fetch and install Java from
 	# the Linux-pkg bundle.
 	#
-	logmust bash -c "echo $jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
+	logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'"
 
-	echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
+	echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO"
 }

packages/masking/config.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git"
+PACKAGE_DEPENDENCIES="adoptopenjdk"
+
+function prepare() {
+	logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb
+}
+
+function build() {
+	export JAVA_HOME
+	JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") ||
+		die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH"
+
+	logmust cd "$WORKDIR/repo"
+
+	#
+	# The "appliance-build-stage0" Jenkins job consumes this file,
+	# along with various other files (e.g. licensing metadata).
+	# Thus, if we don't generate it here, the Jenkins job that
+	# builds the appliance will fail.
+	#
+	# shellcheck disable=SC2016
+	logmust jq -n \
+		--arg h "$(git rev-parse HEAD)" \
+		--arg d "$(date --utc --iso-8601=seconds)" \
+		'{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \
+		>"$WORKDIR/artifacts/metadata.json"
+
+	logmust ./gradlew --no-daemon --stacktrace \
+		-Porg.gradle.configureondemand=false \
+		-PenvironmentName=linuxappliance \
+		clean \
+		generateLicenseReport \
+		:dist:distDeb \
+		:dist:distLicenseReport
+
+	logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/"
+	logmust cp -v \
+		dist/build/reports/dependency-license/* "$WORKDIR/artifacts/"
+}

packages/misc-debs/config.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Copyright 2021 Delphix
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# shellcheck disable=SC2034
+
+DEFAULT_PACKAGE_GIT_URL=none
+SKIP_COPYRIGHTS_CHECK=true
+
+#
+# IMPORTANT NOTE
+# --------------
+#
+# Debian packages (debs) that are not built from source by linux-pkg can be
+# added to this "meta-package". As a general rule, pre-built debs should only
+# be added here when they have been fetched from a trusted third-party
+# package archive.
+#
+# Here are some valid reasons for adding new debs here:
+# - There are bugs with a recent version of a package provided by Ubuntu and
+#   we want to pin an older version of that package.
+# - Ubuntu provides a version of a package that is too old, and the package's
+#   maintainers provide a more recent version of the package. Note that in this
+#   case, you may also look into adding the maintainer's archive to the
+#   linux-package-mirror PPAs list.
+#
+# To add a new deb here, upload that deb to the linux-pkg/misc-debs directory
+# in artifcatory and note the deb's SHA256. Be explicit on where this deb
+# was fetched from and why it was added to this list.
+#
+# When removing debs from this list, you should not remove them from artifactory
+# as they would used when rebuilding older releases.
+#
+
+function fetch() {
+	logmust cd "$WORKDIR/artifacts"
+
+	#
+	# Note about the debs being fetched:
+	# - td-agent was built by the "td-agent" linux-pkg package, but it
+	#   now fails to build due to broken third party dependencies. See
+	#   DLPX-69338 and DLPX-68211.
+	# - unzip was added as a temporary workaround to DLPX-73555.
+	#
+	local debs=(
+		"td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb 84dfa2436039ff2a6312484bd7295ebaf570b5f59f100380b57e68b4800855c4"
+		"unzip_6.0-21ubuntu1_amd64.deb d46069c369ce88c8dd91c52abb8de8d6053606748ef18b3b9bc290fdd8ad2953"
+	)
+
+	local url="http://artifactory.delphix.com/artifactory/linux-pkg/misc-debs"
+
+	echo "Fetched debs:" >BUILD_INFO
+	local entry
+	for entry in "${debs[@]}"; do
+		local deb sha256
+		deb=$(echo "$entry" | awk '{print $1}')
+		sha256=$(echo "$entry" | awk '{print $2}')
+		[[ -n "$deb" && -n "$sha256" ]] || die "Invalid entry '$entry'"
+
+		logmust fetch_file_from_artifactory "$url/$deb" "$sha256"
+
+		echo "$entry" >>BUILD_INFO
+	done
+}
+
+function build() {
+	return
+	# Nothing to do, all the logic is done in fetch().
+}