diff --git a/lib/common.sh b/lib/common.sh index daa64a1d..4b14153c 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -89,7 +89,7 @@ function die() { } function logmust() { - echo Running: "$@" + [[ "$LOGGING" == "false" ]] || echo Running: "$@" >&2 "$@" || die "failed command '$*'" } @@ -470,7 +470,7 @@ function install_pkgs() { echo "Running: sudo env DEBIAN_FRONTEND=noninteractive " \ "apt-get install -y $*" sudo env DEBIAN_FRONTEND=noninteractive apt-get install \ - -y "$@" && return + -y --allow-downgrades "$@" && return echo "apt-get install failed, retrying." sleep 10 done @@ -1116,7 +1116,26 @@ function fetch_kernel_from_artifactory() { url="$url/linux-pkg/linux-prebuilt/${artifactory_deb}" logmust cd "$WORKDIR/artifacts" - logmust wget -nv "$url" + fetch_file_from_artifactory "$url" +} + +function fetch_file_from_artifactory() { + local url="$1" + local sha256_expected="$2" + local file sha256_actual + + [[ -n "$url" ]] || die "url argument is missing." + file="$(basename "$url")" + + logmust wget -nv "$url" -O "$file" + + if [[ -n "$sha256_expected" ]]; then + sha256_actual="$(sha256sum "$file" | awk '{print $1}')" + if [[ "$sha256_expected" != "$sha256_actual" ]]; then + die "SHA256 mismatch. Expected: $sha256_expected," \ + "Actual: $sha256_actual" + fi + fi } # diff --git a/package-lists/build/main.pkgs b/package-lists/build/main.pkgs index 28bffca3..fc00dd00 100644 --- a/package-lists/build/main.pkgs +++ b/package-lists/build/main.pkgs @@ -18,6 +18,8 @@ grub2 libkdumpfile make-jpkg makedumpfile +masking +misc-debs nfs-utils performance-diagnostics ptools @@ -26,4 +28,4 @@ recovery-environment savedump sdb targetcli-fb -td-agent-prebuilt +virtualization diff --git a/packages/adoptopenjdk/config.sh b/packages/adoptopenjdk/config.sh index eb14a272..13ea4401 100755 --- a/packages/adoptopenjdk/config.sh +++ b/packages/adoptopenjdk/config.sh @@ -19,8 +19,9 @@ DEFAULT_PACKAGE_GIT_URL=none PACKAGE_DEPENDENCIES="make-jpkg" -tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" -jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" +_tarfile="OpenJDK8U-jdk_x64_linux_hotspot_8u262b10.tar.gz" +_tarfile_sha256="733755fd649fad6ae91fc083f7e5a5a0b56410fb6ac1815cff29f744b128b1b1" +_jdk_path="/usr/lib/jvm/adoptopenjdk-java8-jdk-amd64" function prepare() { logmust install_pkgs "$DEPDIR"/make-jpkg/*.deb @@ -29,15 +30,15 @@ function prepare() { function fetch() { logmust cd "$WORKDIR/" - local url="http://artifactory.delphix.com/artifactory" + local url="http://artifactory.delphix.com/artifactory/java-binaries/linux/jdk/8/$_tarfile" - logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile" + logmust fetch_file_from_artifactory "$url" "$_tarfile_sha256" } function build() { logmust cd "$WORKDIR/" - logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<'$WORKDIR/artifacts/JDK_PATH'" + logmust bash -c "echo $_jdk_path >'$WORKDIR/artifacts/JDK_PATH'" - echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO" + echo "Tar file: $_tarfile" >"$WORKDIR/artifacts/BUILD_INFO" } diff --git a/packages/java8/config.sh b/packages/java8/config.sh deleted file mode 100644 index 5d03baeb..00000000 --- a/packages/java8/config.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL=none - -tarfile="jdk-8u171-linux-x64.tar.gz" -jdk_path="/usr/lib/jvm/oracle-java8-jdk-amd64" - -function prepare() { - logmust install_pkgs java-package -} - -function fetch() { - logmust cd "$WORKDIR" - local url="http://artifactory.delphix.com/artifactory" - logmust wget -nv "$url/java-binaries/linux/jdk/8/$tarfile" -O "$tarfile" -} - -function build() { - logmust cd "$WORKDIR" - logmust env DEB_BUILD_OPTIONS=nostrip fakeroot make-jpkg "$tarfile" <<'$WORKDIR/artifacts/JDK_PATH'" - - echo "Tar file: $tarfile" >"$WORKDIR/artifacts/BUILD_INFO" -} diff --git a/packages/masking/config.sh b/packages/masking/config.sh new file mode 100644 index 00000000..a4daf383 --- /dev/null +++ b/packages/masking/config.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/masking/dms-core-gate.git" +PACKAGE_DEPENDENCIES="adoptopenjdk" + +function prepare() { + logmust install_pkgs "$DEPDIR"/adoptopenjdk/*.deb +} + +function build() { + export JAVA_HOME + JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || + die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + + logmust cd "$WORKDIR/repo" + + # + # The "appliance-build-stage0" Jenkins job consumes this file, + # along with various other files (e.g. licensing metadata). + # Thus, if we don't generate it here, the Jenkins job that + # builds the appliance will fail. + # + # shellcheck disable=SC2016 + logmust jq -n \ + --arg h "$(git rev-parse HEAD)" \ + --arg d "$(date --utc --iso-8601=seconds)" \ + '{ "dms-core-gate" : { "git-hash" : $h, "date": $d }}' \ + >"$WORKDIR/artifacts/metadata.json" + + logmust ./gradlew --no-daemon --stacktrace \ + -Porg.gradle.configureondemand=false \ + -PenvironmentName=linuxappliance \ + clean \ + generateLicenseReport \ + :dist:distDeb \ + :dist:distLicenseReport + + logmust rsync -av dist/build/distributions/ "$WORKDIR/artifacts/" + logmust cp -v \ + dist/build/reports/dependency-license/* "$WORKDIR/artifacts/" +} diff --git a/packages/misc-debs/config.sh b/packages/misc-debs/config.sh new file mode 100644 index 00000000..5f08ce4b --- /dev/null +++ b/packages/misc-debs/config.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL=none +SKIP_COPYRIGHTS_CHECK=true + +# +# IMPORTANT NOTE +# -------------- +# +# Debian packages (debs) that are not built from source by linux-pkg can be +# added to this "meta-package". As a general rule, pre-built debs should only +# be added here when they have been fetched from a trusted third-party +# package archive. +# +# Here are some valid reasons for adding new debs here: +# - There are bugs with a recent version of a package provided by Ubuntu and +# we want to pin an older version of that package. +# - Ubuntu provides a version of a package that is too old, and the package's +# maintainers provide a more recent version of the package. Note that in this +# case, you may also look into adding the maintainer's archive to the +# linux-package-mirror PPAs list. +# +# To add a new deb here, upload that deb to the linux-pkg/misc-debs directory +# in artifcatory and note the deb's SHA256. Be explicit on where this deb +# was fetched from and why it was added to this list. +# +# When removing debs from this list, you should not remove them from artifactory +# as they would used when rebuilding older releases. +# + +function fetch() { + logmust cd "$WORKDIR/artifacts" + + # + # Note about the debs being fetched: + # - td-agent was built by the "td-agent" linux-pkg package, but it + # now fails to build due to broken third party dependencies. See + # DLPX-69338 and DLPX-68211. + # - unzip was added as a temporary workaround to DLPX-73555. + # + local debs=( + "td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb 84dfa2436039ff2a6312484bd7295ebaf570b5f59f100380b57e68b4800855c4" + "unzip_6.0-21ubuntu1_amd64.deb d46069c369ce88c8dd91c52abb8de8d6053606748ef18b3b9bc290fdd8ad2953" + ) + + local url="http://artifactory.delphix.com/artifactory/linux-pkg/misc-debs" + + echo "Fetched debs:" >BUILD_INFO + local entry + for entry in "${debs[@]}"; do + local deb sha256 + deb=$(echo "$entry" | awk '{print $1}') + sha256=$(echo "$entry" | awk '{print $2}') + [[ -n "$deb" && -n "$sha256" ]] || die "Invalid entry '$entry'" + + logmust fetch_file_from_artifactory "$url/$deb" "$sha256" + + echo "$entry" >>BUILD_INFO + done +} + +function build() { + return + # Nothing to do, all the logic is done in fetch(). +} diff --git a/packages/td-agent-prebuilt/config.sh b/packages/td-agent-prebuilt/config.sh deleted file mode 100644 index 8e5e2156..00000000 --- a/packages/td-agent-prebuilt/config.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Copyright 2019 Delphix -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# shellcheck disable=SC2034 - -DEFAULT_PACKAGE_GIT_URL=none -SKIP_COPYRIGHTS_CHECK=true - -function fetch() { - logmust cd "$WORKDIR/artifacts" - local package="td-agent_3.5.0-delphix-2019.09.18.20_amd64.deb" - local url="http://artifactory.delphix.com/artifactory" - - logmust wget -nv "$url/linux-pkg/td-agent/$package" -O "$package" -} - -function build() { - return - # Nothing to do. See the td-agent package config for the actual build. -} diff --git a/packages/virtualization/config.sh b/packages/virtualization/config.sh new file mode 100644 index 00000000..20c01f8c --- /dev/null +++ b/packages/virtualization/config.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# +# Copyright 2021 Delphix +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# shellcheck disable=SC2034 + +DEFAULT_PACKAGE_GIT_URL="https://gitlab.delphix.com/app/dlpx-app-gate.git" +PACKAGE_DEPENDENCIES="adoptopenjdk crypt-blowfish misc-debs" + +function prepare() { + logmust install_pkgs \ + ant \ + gcc \ + libcairo2 \ + libcurl4-openssl-dev \ + libjbig0 \ + libnss3-dev \ + libnss3-dbg \ + libnss3-tools \ + libpam0g-dev \ + libpixman-1-0 \ + libssl-dev \ + libtiff5 \ + libxcb-render0 \ + libxcb-shm0 \ + python-jira \ + python-requests \ + rsync + + logmust install_pkgs \ + "$DEPDIR"/adoptopenjdk/*.deb \ + "$DEPDIR"/crypt-blowfish/*.deb \ + "$DEPDIR"/misc-debs/unzip_6.0-21ubuntu1_amd64.deb +} + +function build() { + export JAVA_HOME + JAVA_HOME=$(cat "$DEPDIR/adoptopenjdk/JDK_PATH") || + die "Failed to read $DEPDIR/adoptopenjdk/JDK_PATH" + + export LANG + LANG=en_US.UTF-8 + + logmust cd "$WORKDIR/repo" + + # + # The "appliance-build-stage0" Jenkins job consumes this file, + # along with various other files (e.g. licensing metadata). + # Thus, if we don't generate it here, the Jenkins job that + # builds the appliance will fail. + # + # shellcheck disable=SC2016 + logmust jq -n \ + --arg h "$(git rev-parse HEAD)" \ + --arg d "$(date --utc --iso-8601=seconds)" \ + '{ "dlpx-app-gate" : { "git-hash" : $h, "date": $d }}' \ + >"$WORKDIR/artifacts/metadata.json" + + # + # Build the virtualization package + # + logmust cd "$WORKDIR/repo/appliance" + if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then + logmust ant -Ddockerize=true -DbuildJni=true \ + -DhotfixGenDlpxVersion="$DELPHIX_RELEASE_VERSION" \ + all package + else + logmust ant -Ddockerize=true -DbuildJni=true all package + fi + + # + # Publish the virtualization package artifacts + # + logmust cd "$WORKDIR/repo/appliance" + logmust rsync -av packaging/build/distributions/ "$WORKDIR/artifacts/" + logmust cp -vr \ + bin/out/common/com.delphix.common/uem/tars \ + "$WORKDIR/artifacts/hostchecker2" + logmust cp -v \ + server/api/build/api/json-schemas/delphix.json \ + "$WORKDIR/artifacts" + logmust cp -v \ + dist/server/opt/delphix/client/etc/api.ini \ + "$WORKDIR/artifacts" + logmust cp -v \ + packaging/build/reports/dependency-license/* \ + "$WORKDIR/artifacts/" + + # + # Build the "toolkit-devkit" artifacts + # + logmust cd "$WORKDIR/repo/appliance/toolkit" + if [[ -n "$DELPHIX_RELEASE_VERSION" ]]; then + logmust ant \ + -Dversion.number="$DELPHIX_RELEASE_VERSION" \ + toolkit-devkit + else + logmust ant \ + "-Dversion.number=$(date --utc +%Y-%m-%d-%H-%m)" \ + toolkit-devkit + fi + + # + # Publish the "toolkit-devkit" artifacts + # + logmust cd "$WORKDIR/repo/appliance" + logmust mkdir -p "$WORKDIR/artifacts/hostchecker2" + logmust cp -v toolkit/toolkit-devkit.tar "$WORKDIR/artifacts" +} diff --git a/query-packages.sh b/query-packages.sh index b72f81af..8cab51fb 100755 --- a/query-packages.sh +++ b/query-packages.sh @@ -19,6 +19,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 TOP="$PWD" source "$TOP/lib/common.sh" +# +# Disable "logmust" output as that is unnecessary here, and actually +# trips up some of our testing (e.g. verify-query-packages.sh). +# +export LOGGING=false + ALL_OUTPUT_FIELDS=(name git-url dependencies can-update) function usage() { diff --git a/setup.sh b/setup.sh index 03192636..5fd99dd7 100755 --- a/setup.sh +++ b/setup.sh @@ -93,12 +93,14 @@ logmust sudo apt-get update # build dependencies from a control file. # - install_shfmt and shellcheck are needed for - make check - to be able to # make sure style checks are fine. +# - jq is used to generate a JSON formatted metadata file by some packages. # logmust install_pkgs \ debhelper \ devscripts \ equivs \ - shellcheck + shellcheck \ + jq logmust install_shfmt