From bf11bfbb60a5dddf14f1a96e78577f223a2d0fc1 Mon Sep 17 00:00:00 2001 From: Thomas-Apter Date: Mon, 17 Aug 2020 14:57:31 +0100 Subject: [PATCH 01/17] Reformatted testing.md and adding required step make build-devjmstest to Docker tests instructions --- docs/testing.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 07bf3a0e..f0e9e975 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,7 +14,13 @@ There are two main sets of tests: 2. Docker tests, which test a complete Docker image, using the Docker API ### Running the Docker tests -The Docker tests can be run locally on a machine with Docker. For example: +Before running Docker tests, the following command must be ran: + +``` +make build-devjmstest +``` + +The Docker tests can be run locally on a machine with Docker. For example: ``` make test-devserver @@ -25,6 +31,7 @@ You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` ``` MQ_IMAGE_ADVANCEDSERVER=ibm-mqadvanced-server:9.2.0.0-amd64 make test-advancedserver +``` You can pass parameters to `go test` with an environment variable. For example, to run the "TestGoldenPath" test, run the following command: From ecbe7b4f7292af552a0839e740f936c07cc185cd Mon Sep 17 00:00:00 2001 From: Thomas-Apter Date: Mon, 17 Aug 2020 15:56:42 +0100 Subject: [PATCH 02/17] Changed 'ran' to 'run' --- docs/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index f0e9e975..ee25af49 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,7 +14,7 @@ There are two main sets of tests: 2. Docker tests, which test a complete Docker image, using the Docker API ### Running the Docker tests -Before running Docker tests, the following command must be ran: +Before running Docker tests, the following command must be run: ``` make build-devjmstest From 3982cf35175e0eae21aa0e3037e7eb0326e1ebe8 Mon Sep 17 00:00:00 2001 From: Thomas-Apter Date: Tue, 18 Aug 2020 10:10:39 +0100 Subject: [PATCH 03/17] Create advancedserver make target, and update docs to use 'make advancedserver' --- Makefile | 3 +++ docs/testing.md | 7 +------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 5b7af543..b739214f 100644 --- a/Makefile +++ b/Makefile @@ -133,6 +133,9 @@ test-all: build-devjmstest test-devserver test-advancedserver .PHONY: devserver devserver: build-devserver build-devjmstest test-devserver +.PHONY: advancedserver +advancedserver: build-advancedserver test-advancedserver + # Build incubating components .PHONY: incubating incubating: build-explorer diff --git a/docs/testing.md b/docs/testing.md index ee25af49..bbe0d0aa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,12 +14,7 @@ There are two main sets of tests: 2. Docker tests, which test a complete Docker image, using the Docker API ### Running the Docker tests -Before running Docker tests, the following command must be run: - -``` -make build-devjmstest -``` - + The Docker tests can be run locally on a machine with Docker. For example: ``` From e6119202b498798280da7ccabc0a0159eeb9e761 Mon Sep 17 00:00:00 2001 From: Thomas-Apter Date: Tue, 18 Aug 2020 10:13:08 +0100 Subject: [PATCH 04/17] Changed testming.md to use make advancedserver --- docs/testing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index bbe0d0aa..55ded8a8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -18,8 +18,8 @@ There are two main sets of tests: The Docker tests can be run locally on a machine with Docker. For example: ``` -make test-devserver -make test-advancedserver +make devserver +make advancedserver ``` You can specify the image to use directly by using the `MQ_IMAGE_ADVANCEDSERVER` or `MQ_IMAGE_DEVSERVER` variables, for example: From dd88fe54414c69ca84dfeceb64a19b60f978d5f3 Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Tue, 18 Aug 2020 16:49:01 +0530 Subject: [PATCH 05/17] adding zlinux-support --- .travis.yml | 30 +++++++++++++++++++----------- travis-build-scripts/build.sh | 2 +- travis-build-scripts/push.sh | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index a534e912..6c2104e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,13 +36,21 @@ env: jobs: include: - - stage: build - name: "Basic build" + - stage: build-amd64 if: branch != private-master AND tag IS blank + name: "Basic AMD64 build" os: linux env: - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64 script: bash -e travis-build-scripts/run.sh + - stage: build-s390x + if: branch != private-master AND tag IS blank + name: "Basic S390X build" + os: linux-s390 + env: + - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics" + - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X + script: travis_wait 60 bash -e travis-build-scripts/run.sh - if: branch = private-master OR tag =~ ^release-candidate* name: "Multi-Arch AMD64 build" os: linux @@ -60,15 +68,15 @@ jobs: # # - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_PPC64LE # - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_PPC64LE # script: bash -e travis-build-scripts/run.sh - # - if: branch = private-master OR tag =~ ^release-candidate* - # name: "Multi-Arch S390X build" - # os: linux-s390 - # env: - # - BUILD_ALL=true - # - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics" - # # - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_S390X - # - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X - # script: bash -e travis-build-scripts/run.sh + - if: branch = private-master OR tag =~ ^release-candidate* + name: "Multi-Arch S390X build" + os: linux-s390 + env: + - BUILD_ALL=true + - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics" + - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_S390X + - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X + script: bash -e travis-build-scripts/run.sh before_install: - make install-build-deps diff --git a/travis-build-scripts/build.sh b/travis-build-scripts/build.sh index c9fae5a4..cd634fd3 100755 --- a/travis-build-scripts/build.sh +++ b/travis-build-scripts/build.sh @@ -23,7 +23,7 @@ echo 'Building Developer image...' && echo -en 'travis_fold:start:build-devserve make build-devserver echo -en 'travis_fold:end:build-devserver\\r' if [ "$BUILD_ALL" = true ] ; then - if [ "$ARCH" = "amd64" ] ; then + if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then echo 'Building Production image...' && echo -en 'travis_fold:start:build-advancedserver\\r' make build-advancedserver echo -en 'travis_fold:end:build-advancedserver\\r' diff --git a/travis-build-scripts/push.sh b/travis-build-scripts/push.sh index 55154bb0..5165cc2d 100755 --- a/travis-build-scripts/push.sh +++ b/travis-build-scripts/push.sh @@ -32,7 +32,7 @@ function push_developer { } function push_production { - if [ "$ARCH" = "amd64" ] ; then + if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then echo 'Pushing Production image...' && echo -en 'travis_fold:start:push-advancedserver\\r' make push-advancedserver echo -en 'travis_fold:end:push-advancedserver\\r' From 1b9f80a9af37d521f18c29e7dd6d1e3404947555 Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Tue, 18 Aug 2020 17:22:26 +0530 Subject: [PATCH 06/17] adding zlinux-support --- travis-build-scripts/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index a54b170d..f9627996 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -20,14 +20,14 @@ echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\ make test-devserver echo -en 'travis_fold:end:test-devserver\\r' if [ "$BUILD_ALL" = true ] ; then - if [ "$ARCH" = "amd64" ] ; then + if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' make test-advancedserver echo -en 'travis_fold:end:test-advancedserver\\r' fi fi echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r' -if [ "$ARCH" = "amd64" ] ; then +if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then make gosec else echo "Gosec not available on ppc64le/s390x...skipping gosec scan" From 5a9f1d13e4f47cca4bbf4444f4ac01ab28db4f1d Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Tue, 18 Aug 2020 19:03:54 +0530 Subject: [PATCH 07/17] addressing review comments --- .travis.yml | 8 -------- travis-build-scripts/test.sh | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c2104e8..0127702a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,14 +43,6 @@ jobs: env: - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64 script: bash -e travis-build-scripts/run.sh - - stage: build-s390x - if: branch != private-master AND tag IS blank - name: "Basic S390X build" - os: linux-s390 - env: - - TEST_OPTS_DOCKER="-run TestGoldenPathWithMetrics" - - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X - script: travis_wait 60 bash -e travis-build-scripts/run.sh - if: branch = private-master OR tag =~ ^release-candidate* name: "Multi-Arch AMD64 build" os: linux diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index f9627996..036bf045 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -27,7 +27,7 @@ if [ "$BUILD_ALL" = true ] ; then fi fi echo 'Running gosec scan...' && echo -en 'travis_fold:start:gosec-scan\\r' -if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then +if [ "$ARCH" = "amd64" ] ; then make gosec else echo "Gosec not available on ppc64le/s390x...skipping gosec scan" From 5403c7af82a45904d7a5ab93a1fa0165d91eecf8 Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Thu, 20 Aug 2020 15:34:11 +0530 Subject: [PATCH 08/17] fat-manifest updates fat-manifest updates fat-manifest updates Making create-manifest-list.sh executable fixing travis failure fixing fat-manifests fat-manifest fat-manifest fat-manifest updates fat-manifests fat-manifest updates manifest-list updates fat-manifest updating fat-manifests fat-manifests --- .travis.yml | 15 ++-- Makefile | 40 +++++++++ docker-builds/skopeo/Dockerfile | 17 ++++ travis-build-scripts/create-manifest-list.sh | 90 ++++++++++++++++++++ 4 files changed, 157 insertions(+), 5 deletions(-) create mode 100644 docker-builds/skopeo/Dockerfile create mode 100755 travis-build-scripts/create-manifest-list.sh diff --git a/.travis.yml b/.travis.yml index 0127702a..065bdc8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,14 +36,15 @@ env: jobs: include: - - stage: build-amd64 + - stage: basic-build if: branch != private-master AND tag IS blank name: "Basic AMD64 build" os: linux env: - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_AMD64 script: bash -e travis-build-scripts/run.sh - - if: branch = private-master OR tag =~ ^release-candidate* + - stage: build + if: branch = private-master OR tag =~ ^release-candidate* name: "Multi-Arch AMD64 build" os: linux env: @@ -60,7 +61,8 @@ jobs: # # - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_PPC64LE # - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_PPC64LE # script: bash -e travis-build-scripts/run.sh - - if: branch = private-master OR tag =~ ^release-candidate* + - stage: build + if: branch = private-master OR tag =~ ^release-candidate* name: "Multi-Arch S390X build" os: linux-s390 env: @@ -69,7 +71,10 @@ jobs: - MQ_ARCHIVE_REPOSITORY=$MQ_920_ARCHIVE_REPOSITORY_S390X - MQ_ARCHIVE_REPOSITORY_DEV=$MQ_920_ARCHIVE_REPOSITORY_DEV_S390X script: bash -e travis-build-scripts/run.sh - + - stage: push-manifest + if: branch = private-master OR tag =~ ^release-candidate* + name: "Push Manifest-list to registry" + script: make push-manifest before_install: - make install-build-deps - make install-credential-helper @@ -80,4 +85,4 @@ install: before_script: echo nothing after_success: - - make lint + - make lint \ No newline at end of file diff --git a/Makefile b/Makefile index b739214f..c8032ef9 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,7 @@ BASE_IMAGE_TAG=$(lastword $(subst /, ,$(subst :,-,$(BASE_IMAGE)))) MQ_IMAGE_DEVSERVER_BASE=mqadvanced-server-dev-base # Docker image name to use for JMS tests DEV_JMS_IMAGE=mq-dev-jms-test +MQ_DELIVERY_REGISTRY_NAMESPACE=kd # Variables for versioning IMAGE_REVISION=$(shell git rev-parse HEAD) IMAGE_SOURCE=$(shell git config --get remote.origin.url) @@ -117,6 +118,25 @@ endif MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) +#setup variables for fat-manifests +ifneq "$(RELEASE)" "$(EMPTY)" + MQ_MANIFEST_TAG=$(MQ_VERSION)-$(RELEASE) + MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) + MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) + MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 + MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x + MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 + MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-S390X +else + MQ_MANIFEST_TAG=$(MQ_VERSION) + MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) + MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) + MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 + MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x + MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 + MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-s390x +endif + ############################################################################### # Build targets ############################################################################### @@ -345,6 +365,26 @@ pull-devserver: $(COMMAND) pull $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(COMMAND) tag $(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEV_FULL_RELEASE_NAME) $(MQ_IMAGE_DEVSERVER)\:$(MQ_TAG) +.PHONY: push-manifest +push-manifest: build-skopeo-container + $(info $(SPACER)$(shell printf $(TITLE)"** Determining the image digests **"$(END))) + $(eval MQ_IMAGE_DEVSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux --override-arch s390x inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_AMD64) | jq -r .Digest)) + $(eval MQ_IMAGE_DEVSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_DEVSERVER_S390X) | jq -r .Digest)) + $(eval MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_AMD64) | jq -r .Digest)) + $(eval MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST=$(shell $(COMMAND) run skopeo:latest --override-os linux inspect --creds $(MQ_ARCHIVE_REPOSITORY_USER):$(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) docker://$(MQ_IMAGE_ADVANCEDSERVER_S390X) | jq -r .Digest)) + $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_AMD64) has a digest of $(MQ_IMAGE_DEVSERVER_AMD64_DIGEST)**"$(END))) + $(info $(shell printf "** Determined the built $(MQ_IMAGE_DEVSERVER_S390X) has a digest of $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)**"$(END))) + $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_AMD64) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST)**"$(END))) + $(info $(shell printf "** Determined the built $(MQ_IMAGE_ADVANCEDSERVER_S390X) has a digest of $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)**"$(END))) + $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_DEVSERVER_MANIFEST)**"$(END))) + echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_DEVSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_DEVSERVER_AMD64_DIGEST) $(MQ_IMAGE_DEVSERVER_S390X_DIGEST)" $(END)) + $(info $(shell printf "** Calling script to create fat-manifest for $(MQ_IMAGE_ADVANCEDSERVER_MANIFEST)**"$(END))) + echo $(shell ./travis-build-scripts/create-manifest-list.sh -r $(MQ_DELIVERY_REGISTRY_HOSTNAME) -n $(MQ_DELIVERY_REGISTRY_NAMESPACE) -i $(MQ_IMAGE_ADVANCEDSERVER) -t $(MQ_MANIFEST_TAG) -u $(MQ_ARCHIVE_REPOSITORY_USER) -p $(MQ_ARCHIVE_REPOSITORY_CREDENTIAL) -d "$(MQ_IMAGE_ADVANCEDSERVER_AMD64_DIGEST) $(MQ_IMAGE_ADVANCEDSERVER_S390X_DIGEST)" $(END)) + +.PHONY: build-skopeo-container +build-skopeo-container: + $(COMMAND) images | grep -q "skopeo"; if [ $$? != 0 ]; then docker build -t skopeo:latest ./docker-builds/skopeo/; fi + .PHONY: clean clean: rm -rf ./coverage diff --git a/docker-builds/skopeo/Dockerfile b/docker-builds/skopeo/Dockerfile new file mode 100644 index 00000000..7447e463 --- /dev/null +++ b/docker-builds/skopeo/Dockerfile @@ -0,0 +1,17 @@ +# © Copyright IBM Corporation 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM fedora:32 +RUN yum install skopeo -y -qq +ENTRYPOINT [ "skopeo" ] diff --git a/travis-build-scripts/create-manifest-list.sh b/travis-build-scripts/create-manifest-list.sh new file mode 100755 index 00000000..830082a0 --- /dev/null +++ b/travis-build-scripts/create-manifest-list.sh @@ -0,0 +1,90 @@ +#!/bin/bash + +# © Copyright IBM Corporation 2020 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage=" +Usage: create-image-manifest.sh -r hyc-mq-container-team-docker-local.artifactory.swg-devops.com -n foo -i ibm-mqadvanced-server-dev -t test -d \"sha256:038ad492532b099c324b897ce9da31ae0be312a1d0063f6456f2e3143cc4f4b8 sha256:754f466cf2cfc5183ac705689ce6720f27fecd07c97970ba3ec48769acba067d\" + +Where: +-r - The image registry hostname +-n - The image registry namespace +-i - The image name +-t - The desired top level manifest tag +-d - A space separated list of sha256 image digests to be included +" + +GREEN="\033[32m" +RED="\033[31m" +BLUE="\033[34m" +PURPLE="\033[35m" +AQUA="\033[36m" + +END="\033[0m" + +UNDERLINE="\033[4m" +BOLD="\033[1m" +ITALIC="\033[3m" +TITLE=${BLUE}${BOLD}${UNDERLINE} +STEPTITLE=${BLUERIGHTARROW}" "${BOLD}${ITALIC} +SUBSTEPTITLE=${MINIARROW}${MINIARROW}${MINIARROW}" "${ITALIC} +RIGHTARROW="\xE2\x96\xB6" +MINIARROW="\xE2\x96\xBB" +BLUERIGHTARROW=${BLUE}${RIGHTARROW}${END} +GREENRIGHTARROW=${GREEN}${RIGHTARROW}${END} + +ERROR=${RED} + +TICK="\xE2\x9C\x94" +CROSS="\xE2\x9C\x97" +GREENTICK=${GREEN}${TICK}${END} +REDCROSS=${RED}${CROSS}${END} + + +SPACER="\n\n" + +while getopts r:n:i:t:d:h:u:p: flag +do + case "${flag}" in + r) REGISTRY=${OPTARG};; + n) NAMESPACE=${OPTARG};; + i) IMAGE=${OPTARG};; + t) TAG=${OPTARG};; + d) DIGESTS=${OPTARG};; + u) USER=${OPTARG};; + p) CREDENTIAL=${OPTARG};; + esac +done + +if [[ -z $REGISTRY || -z $NAMESPACE || -z $IMAGE || -z $TAG || -z $DIGESTS ]] ; then + printf "${REDCROSS} ${ERROR}Missing parameter!${END}\n" + printf "${ERROR}$usage${END}\n" + exit 1 +fi + +# Docker CLI manifest commands require experimental features to be turned on +export DOCKER_CLI_EXPERIMENTAL=enabled + +MANIFESTS="" +for digest in $DIGESTS ; do \ + MANIFESTS+=" $REGISTRY/$NAMESPACE/$IMAGE@$digest" +done + +docker login $REGISTRY -u $USER -p $CREDENTIAL +docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG --amend $MANIFESTS > /dev/null +MANIFEST_DIGEST=$(docker manifest push $REGISTRY/$NAMESPACE/$IMAGE:$TAG) + +echo $MANIFEST_DIGEST \ No newline at end of file From eb4a734ad0fc83e752a619a10a91f35b98411bcf Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Fri, 21 Aug 2020 13:53:48 +0530 Subject: [PATCH 09/17] fat-manifest updates --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index c8032ef9..a36e29c5 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,6 @@ BASE_IMAGE_TAG=$(lastword $(subst /, ,$(subst :,-,$(BASE_IMAGE)))) MQ_IMAGE_DEVSERVER_BASE=mqadvanced-server-dev-base # Docker image name to use for JMS tests DEV_JMS_IMAGE=mq-dev-jms-test -MQ_DELIVERY_REGISTRY_NAMESPACE=kd # Variables for versioning IMAGE_REVISION=$(shell git rev-parse HEAD) IMAGE_SOURCE=$(shell git config --get remote.origin.url) From f951bfaffacf832979f44054e2e0ac89b62d68ab Mon Sep 17 00:00:00 2001 From: KIRAN DARBHA Date: Fri, 21 Aug 2020 14:40:42 +0530 Subject: [PATCH 10/17] addressing review comments --- Makefile | 26 +++++++------------- travis-build-scripts/create-manifest-list.sh | 6 ++--- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index a36e29c5..875ee7a2 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,8 @@ REGISTRY_USER ?= REGISTRY_PASS ?= # ARCH is the platform architecture (e.g. amd64, ppc64le or s390x) ARCH ?= $(if $(findstring x86_64,$(shell uname -m)),amd64,$(shell uname -m)) +# Tag to use for fat-manifest +MQ_MANIFEST_TAG=$(MQ_VERSION) ############################################################################### # Other variables @@ -113,28 +115,18 @@ endif ifneq "$(RELEASE)" "$(EMPTY)" MQ_TAG=$(MQ_VERSION)-$(RELEASE)-$(ARCH) EXTRA_LABELS=--label release=$(RELEASE) + MQ_MANIFEST_TAG=$(MQ_VERSION)-$(RELEASE) endif MQ_IMAGE_FULL_RELEASE_NAME=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) MQ_IMAGE_DEV_FULL_RELEASE_NAME=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) #setup variables for fat-manifests -ifneq "$(RELEASE)" "$(EMPTY)" - MQ_MANIFEST_TAG=$(MQ_VERSION)-$(RELEASE) - MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) - MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) - MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 - MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x - MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 - MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-S390X -else - MQ_MANIFEST_TAG=$(MQ_VERSION) - MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) - MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) - MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 - MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x - MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 - MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-s390x -endif +MQ_IMAGE_DEVSERVER_MANIFEST=$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG) +MQ_IMAGE_ADVANCEDSERVER_MANIFEST=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG) +MQ_IMAGE_DEVSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-amd64 +MQ_IMAGE_DEVSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_DEVSERVER):$(MQ_MANIFEST_TAG)-s390x +MQ_IMAGE_ADVANCEDSERVER_AMD64=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-amd64 +MQ_IMAGE_ADVANCEDSERVER_S390X=$(MQ_DELIVERY_REGISTRY_FULL_PATH)/$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_MANIFEST_TAG)-s390x ############################################################################### # Build targets diff --git a/travis-build-scripts/create-manifest-list.sh b/travis-build-scripts/create-manifest-list.sh index 830082a0..788e5ef0 100755 --- a/travis-build-scripts/create-manifest-list.sh +++ b/travis-build-scripts/create-manifest-list.sh @@ -84,7 +84,7 @@ for digest in $DIGESTS ; do \ done docker login $REGISTRY -u $USER -p $CREDENTIAL -docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG --amend $MANIFESTS > /dev/null -MANIFEST_DIGEST=$(docker manifest push $REGISTRY/$NAMESPACE/$IMAGE:$TAG) +docker manifest create $REGISTRY/$NAMESPACE/$IMAGE:$TAG $MANIFESTS > /dev/null +MANIFEST_DIGEST=$(docker manifest push --purge $REGISTRY/$NAMESPACE/$IMAGE:$TAG) -echo $MANIFEST_DIGEST \ No newline at end of file +echo $MANIFEST_DIGEST From 98f01a40a74080f6802cac8b1d6773c4eaca7bd0 Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Fri, 4 Sep 2020 16:22:13 +0100 Subject: [PATCH 11/17] Updated to go 1.13.15, switched to community goloang image --- .travis.yml | 2 +- Dockerfile-server | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 065bdc8a..ca2aae3c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ sudo: required language: go go: - - "1.12" + - "1.13.15" services: - docker diff --git a/Dockerfile-server b/Dockerfile-server index c8f0f049..57875454 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -13,18 +13,20 @@ # limitations under the License. ARG BASE_IMAGE=registry.redhat.io/ubi8/ubi-minimal -ARG BASE_TAG=8.2-301.1593113563 +ARG BASE_TAG=8.2-345 +ARG GO_WORKDIR=/go/src/github.com/ibm-messaging/mq-container +ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/9.2.0.0-IBM-MQ-Advanced-for-Developers-Non-Install-LinuxX64.tar.gz" ############################################################################### # Build stage to build Go code ############################################################################### -FROM registry.redhat.io/ubi8/go-toolset:1.13.4-22 as builder -# FROM docker.io/centos/go-toolset-7-centos7 as builder +FROM golang:1.13.15 as builder # The URL to download the MQ installer from in tar.gz format -# This assumes an archive containing the MQ RPM install packages -ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev912_linux_x86-64.tar.gz" +# This assumes an archive containing the MQ Non-Install packages +ARG MQ_URL ARG IMAGE_REVISION="Not specified" ARG IMAGE_SOURCE="Not specified" ARG IMAGE_TAG="Not specified" +ARG GO_WORKDIR USER 0 COPY install-mq.sh /usr/local/bin/ RUN mkdir /opt/mqm \ @@ -32,13 +34,12 @@ RUN mkdir /opt/mqm \ && sleep 1 \ && INSTALL_SDK=1 install-mq.sh \ && chown -R 1001:root /opt/mqm/* -WORKDIR /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/ +WORKDIR $GO_WORKDIR/ COPY cmd/ ./cmd COPY internal/ ./internal COPY pkg/ ./pkg COPY vendor/ ./vendor -ENV PATH="${PATH}:/opt/rh/go-toolset-1.11/root/usr/bin" \ - CGO_CFLAGS="-I/opt/mqm/inc/" \ +ENV CGO_CFLAGS="-I/opt/mqm/inc/" \ CGO_LDFLAGS_ALLOW="-Wl,-rpath.*" RUN go build -ldflags "-X \"main.ImageCreated=$(date --iso-8601=seconds)\" -X \"main.ImageRevision=$IMAGE_REVISION\" -X \"main.ImageSource=$IMAGE_SOURCE\" -X \"main.ImageTag=$IMAGE_TAG\"" ./cmd/runmqserver/ RUN go build ./cmd/chkmqready/ @@ -58,11 +59,10 @@ RUN go vet ./cmd/... ./internal/... ############################################################################### FROM $BASE_IMAGE:$BASE_TAG AS mq-server # The MQ packages to install - see install-mq.sh for default value -ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/mqadv_dev914_linux_x86-64.tar.gz" -ARG MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesWeb*.rpm MQSeriesAMS-*.rpm" -#ARG MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-web ibmmq-ams" +ARG MQ_URL ARG BASE_IMAGE ARG BASE_TAG +ARG GO_WORKDIR LABEL summary="IBM MQ Advanced Server" LABEL description="Simplify, accelerate and facilitate the reliable exchange of data with a security-rich messaging solution — trusted by the world’s most successful enterprises" LABEL vendor="IBM" @@ -89,8 +89,8 @@ RUN env \ # Create a directory for runtime data from runmqserver RUN mkdir -p /run/runmqserver \ && chown 1001:root /run/runmqserver -COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/runmqserver /usr/local/bin/ -COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/chkmq* /usr/local/bin/ +COPY --from=builder $GO_WORKDIR/runmqserver /usr/local/bin/ +COPY --from=builder $GO_WORKDIR/chkmq* /usr/local/bin/ COPY NOTICES.txt /opt/mqm/licenses/notices-container.txt # Copy web XML files COPY web /etc/mqm/web @@ -120,6 +120,7 @@ ENTRYPOINT ["runmqserver"] FROM mq-server AS mq-dev-server ARG BASE_IMAGE ARG BASE_TAG +ARG GO_WORKDIR # Enable MQ developer default configuration ENV MQ_DEV=true LABEL summary="IBM MQ Advanced for Developers Server" @@ -134,7 +135,7 @@ LABEL io.k8s.description="Simplify, accelerate and facilitate the reliable excha LABEL base-image=$BASE_IMAGE LABEL base-image-release=$BASE_TAG USER 0 -COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/amqpas* /opt/mqm/lib64/ +COPY --from=builder $GO_WORKDIR/amqpas* /opt/mqm/lib64/ COPY etc/mqm/*.ini /etc/mqm/ COPY etc/mqm/mq.htpasswd /etc/mqm/ RUN chmod 0660 /etc/mqm/mq.htpasswd @@ -145,7 +146,7 @@ RUN chmod u+x /usr/local/bin/install-extra-packages.sh \ # Create a directory for runtime data from runmqserver RUN mkdir -p /run/runmqdevserver \ && chown 1001:root /run/runmqdevserver -COPY --from=builder /opt/app-root/src/go/src/github.com/ibm-messaging/mq-container/runmqdevserver /usr/local/bin/ +COPY --from=builder $GO_WORKDIR/runmqdevserver /usr/local/bin/ # Copy template files COPY incubating/mqadvanced-server-dev/*.tpl /etc/mqm/ # Copy web XML files for default developer configuration From e7fd10e7e6cac6cc79ed1593109fdc1b5f3aa3fa Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Mon, 14 Sep 2020 09:54:03 +0100 Subject: [PATCH 12/17] Updated to latest UBI version --- Dockerfile-server | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-server b/Dockerfile-server index 57875454..70c9e548 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -13,7 +13,7 @@ # limitations under the License. ARG BASE_IMAGE=registry.redhat.io/ubi8/ubi-minimal -ARG BASE_TAG=8.2-345 +ARG BASE_TAG=8.2-349 ARG GO_WORKDIR=/go/src/github.com/ibm-messaging/mq-container ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/9.2.0.0-IBM-MQ-Advanced-for-Developers-Non-Install-LinuxX64.tar.gz" ############################################################################### From 4fa771436155ce6b92c8968477a464894d1d505d Mon Sep 17 00:00:00 2001 From: Luke J Powlett Date: Tue, 15 Sep 2020 10:55:37 +0100 Subject: [PATCH 13/17] MQ container 9.2.0.0-r2 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index ca2aae3c..0f4b8554 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ go_import_path: "github.com/ibm-messaging/mq-container" env: global: - - RELEASE="" + - RELEASE="r2" jobs: include: @@ -85,4 +85,4 @@ install: before_script: echo nothing after_success: - - make lint \ No newline at end of file + - make lint From 95be35d24676dbf3bc2c552f7879e068652912ca Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 2 Nov 2020 12:03:12 +0000 Subject: [PATCH 14/17] Don't use setuid on chkmq* Also add new tests for chkmqhealthy and privileges --- Dockerfile-server | 2 +- Makefile | 11 ++++-- cmd/chkmqhealthy/main.go | 4 +- test/docker/docker_api_test.go | 43 +++++++++++++++++++++- test/docker/docker_api_test_util.go | 57 ++++++++++++++++++----------- 5 files changed, 87 insertions(+), 30 deletions(-) diff --git a/Dockerfile-server b/Dockerfile-server index 70c9e548..5e20b9e8 100644 --- a/Dockerfile-server +++ b/Dockerfile-server @@ -97,7 +97,7 @@ COPY web /etc/mqm/web COPY etc/mqm/*.tpl /etc/mqm/ RUN chmod ug+x /usr/local/bin/runmqserver \ && chown 1001:root /usr/local/bin/*mq* \ - && chmod ug+xs /usr/local/bin/chkmq* \ + && chmod ug+x /usr/local/bin/chkmq* \ && chown -R 1001:root /etc/mqm/* \ && install --directory --mode 2775 --owner 1001 --group root /run/runmqserver \ && touch /run/termination-log \ diff --git a/Makefile b/Makefile index 875ee7a2..7b9d537f 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,9 @@ MQ_ARCHIVE_DEV ?= $(MQ_VERSION)-IBM-MQ-Advanced-for-Developers-Non-Install-$(MQ_ # MQ_SDK_ARCHIVE specifies the archive to use for building the golang programs. Defaults vary on developer or advanced. MQ_SDK_ARCHIVE ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests -TEST_OPTS_DOCKER ?= +TEST_OPTS_DOCKER ?= +# Timeout for the Docker tests +TEST_TIMEOUT_DOCKER ?= 30m # MQ_IMAGE_ADVANCEDSERVER is the name of the built MQ Advanced image MQ_IMAGE_ADVANCEDSERVER ?=ibm-mqadvanced-server # MQ_IMAGE_DEVSERVER is the name of the built MQ Advanced for Developers image @@ -73,7 +75,7 @@ MQ_ARCHIVE_DEV_TYPE=Linux BUILD_SERVER_CONTAINER=build-server # NUM_CPU is the number of CPUs available to Docker. Used to control how many # test run in parallel -NUM_CPU = $(or $(shell docker info --format "{{ .NCPU }}"),2) +NUM_CPU ?= $(or $(shell docker info --format "{{ .NCPU }}"),2) # BASE_IMAGE_TAG is a normalized version of BASE_IMAGE, suitable for use in a Docker tag BASE_IMAGE_TAG=$(lastword $(subst /, ,$(subst :,-,$(BASE_IMAGE)))) #BASE_IMAGE_TAG=$(subst /,-,$(subst :,-,$(BASE_IMAGE))) @@ -183,7 +185,7 @@ test-unit: test-advancedserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) on $(shell docker --version)"$(END))) docker inspect $(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER):$(MQ_TAG) EXPECTED_LICENSE=Production go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) $(TEST_OPTS_DOCKER) .PHONY: build-devjmstest build-devjmstest: @@ -193,8 +195,9 @@ build-devjmstest: .PHONY: test-devserver test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG) on $(shell docker --version)"$(END))) + docker inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) -tags mqdev $(TEST_OPTS_DOCKER) .PHONY: coverage coverage: diff --git a/cmd/chkmqhealthy/main.go b/cmd/chkmqhealthy/main.go index 623a554c..7132bda5 100644 --- a/cmd/chkmqhealthy/main.go +++ b/cmd/chkmqhealthy/main.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2019 +© Copyright IBM Corporation 2017, 2020 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -36,11 +36,11 @@ func queueManagerHealthy() (bool, error) { cmd := exec.Command("dspmq", "-n", "-m", name) // Run the command and wait for completion out, err := cmd.CombinedOutput() + fmt.Printf("%s", out) if err != nil { fmt.Println(err) return false, err } - fmt.Printf("%s", out) if !strings.Contains(string(out), "(RUNNING)") && !strings.Contains(string(out), "(RUNNING AS STANDBY)") && !strings.Contains(string(out), "(STARTING)") { return false, nil } diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index bd41c336..daba66d1 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -1,5 +1,5 @@ /* -© Copyright IBM Corporation 2017, 2019 +© Copyright IBM Corporation 2017, 2020 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1433,3 +1433,44 @@ func TestTraceStrmqm(t *testing.T) { t.Fatalf("No trace files found in trace directory /var/mqm/trace. RC=%d.", rc) } } + +// utilTestHealthCheck is used by TestHealthCheck* to run a container with +// privileges enabled or disabled. Otherwise the same as the golden path tests. +func utilTestHealthCheck(t *testing.T, nonewpriv bool) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + } + hostConfig := getDefaultHostConfig(t, cli) + hostConfig.SecurityOpt = append(hostConfig.SecurityOpt, fmt.Sprintf("no-new-privileges:%v", nonewpriv)) + id := runContainerWithHostConfig(t, cli, &containerConfig, hostConfig) + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + rc, out := execContainer(t, cli, id, "", []string{"chkmqhealthy"}) + t.Log(out) + if rc != 0 { + t.Errorf("Expected chkmqhealthy to return with exit code 0; got \"%v\"", rc) + t.Logf("Output from chkmqhealthy:\n%v", out) + } + // Stop the container cleanly + stopContainer(t, cli, id) +} + +// TestHealthCheckWithNoNewPrivileges tests golden path start/stop plus +// chkmqhealthy, when running in a container where no new privileges are +// allowed (i.e. setuid is disabled) +func TestHealthCheckWithNoNewPrivileges(t *testing.T) { + utilTestHealthCheck(t, true) +} + +// TestHealthCheckWithNoNewPrivileges tests golden path start/stop plus +// chkmqhealthy when running in a container where new privileges are +// allowed (i.e. setuid is allowed) +// See https://github.com/ibm-messaging/mq-container/issues/428 +func TestHealthCheckWithNewPrivileges(t *testing.T) { + utilTestHealthCheck(t, false) +} diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index c2c589cc..597d1552 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -267,20 +267,8 @@ func generateRandomUID() string { return fmt.Sprint(rand.Intn(max-min) + min) } -// runContainerWithPorts creates and starts a container, exposing the specified ports on the host. -// If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE -// environment variable. -func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *container.Config, ports []int) string { - if containerConfig.Image == "" { - containerConfig.Image = imageName() - } - // Always run as a random user, unless the test has specified otherwise - if containerConfig.User == "" { - containerConfig.User = generateRandomUID() - } - // if coverage - containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov") - containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t)) +// getDefaultHostConfig creates a HostConfig and populates it with the defaults used in testing +func getDefaultHostConfig(t *testing.T, cli *client.Client) *container.HostConfig { hostConfig := container.HostConfig{ Binds: []string{ coverageBind(t), @@ -299,17 +287,25 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co } else { t.Logf("Detected MQ Advanced image - dropping all capabilities") } - for _, p := range ports { - port := nat.Port(fmt.Sprintf("%v/tcp", p)) - hostConfig.PortBindings[port] = []nat.PortBinding{ - { - HostIP: "0.0.0.0", - }, - } + return &hostConfig +} + +// runContainerWithHostConfig creates and starts a container, using the supplied HostConfig. +// Note that a default HostConfig can be created using getDefaultHostConfig. +func runContainerWithHostConfig(t *testing.T, cli *client.Client, containerConfig *container.Config, hostConfig *container.HostConfig) string { + if containerConfig.Image == "" { + containerConfig.Image = imageName() + } + // Always run as a random user, unless the test has specified otherwise + if containerConfig.User == "" { + containerConfig.User = generateRandomUID() } + // if coverage + containerConfig.Env = append(containerConfig.Env, "COVERAGE_FILE="+t.Name()+".cov") + containerConfig.Env = append(containerConfig.Env, "EXIT_CODE_FILE="+getExitCodeFilename(t)) networkingConfig := network.NetworkingConfig{} t.Logf("Running container (%s)", containerConfig.Image) - ctr, err := cli.ContainerCreate(context.Background(), containerConfig, &hostConfig, &networkingConfig, t.Name()) + ctr, err := cli.ContainerCreate(context.Background(), containerConfig, hostConfig, &networkingConfig, t.Name()) if err != nil { t.Fatal(err) } @@ -317,6 +313,22 @@ func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *co return ctr.ID } +// runContainerWithPorts creates and starts a container, exposing the specified ports on the host. +// If no image is specified in the container config, then the image name is retrieved from the TEST_IMAGE +// environment variable. +func runContainerWithPorts(t *testing.T, cli *client.Client, containerConfig *container.Config, ports []int) string { + hostConfig := getDefaultHostConfig(t, cli) + for _, p := range ports { + port := nat.Port(fmt.Sprintf("%v/tcp", p)) + hostConfig.PortBindings[port] = []nat.PortBinding{ + { + HostIP: "0.0.0.0", + }, + } + } + return runContainerWithHostConfig(t, cli, containerConfig, hostConfig) +} + // runContainer creates and starts a container. If no image is specified in // the container config, then the image name is retrieved from the TEST_IMAGE // environment variable. @@ -526,6 +538,7 @@ func waitForContainer(t *testing.T, cli *client.Client, ID string, timeout time. // execContainer runs a command in a running container, and returns the exit code and output func execContainer(t *testing.T, cli *client.Client, ID string, user string, cmd []string) (int, string) { + t.Logf("Running command: %v", cmd) config := types.ExecConfig{ User: user, Privileged: false, From 9fba096d772c04de5d4c9e676c978d466c64b507 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Wed, 4 Nov 2020 16:26:13 +0000 Subject: [PATCH 15/17] Use travis_wait to avoid 10 min timeout for tests --- travis-build-scripts/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index 036bf045..ac78c7d5 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -17,12 +17,12 @@ set -e echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' -make test-devserver +travis_wait make test-devserver echo -en 'travis_fold:end:test-devserver\\r' if [ "$BUILD_ALL" = true ] ; then if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' - make test-advancedserver + travis_wait make test-advancedserver echo -en 'travis_fold:end:test-advancedserver\\r' fi fi From f50ed4d9c7bce8fc310eabe76081772d346c9250 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 10 Nov 2020 14:19:57 +0000 Subject: [PATCH 16/17] Use verbose test output from Travis --- travis-build-scripts/test.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index ac78c7d5..9eebf84d 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -16,13 +16,16 @@ set -e +# Use verbose test output +TEST_OPTS_DOCKER="-v" + echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' -travis_wait make test-devserver +make test-devserver echo -en 'travis_fold:end:test-devserver\\r' if [ "$BUILD_ALL" = true ] ; then if [[ "$ARCH" = "amd64" || "$ARCH" = "s390x" ]] ; then echo 'Testing Production image...' && echo -en 'travis_fold:start:test-advancedserver\\r' - travis_wait make test-advancedserver + make test-advancedserver echo -en 'travis_fold:end:test-advancedserver\\r' fi fi From 829d9dfc585cde960f856fd4dc1981af957d5142 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Tue, 10 Nov 2020 16:29:32 +0000 Subject: [PATCH 17/17] Export variable --- travis-build-scripts/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-build-scripts/test.sh b/travis-build-scripts/test.sh index 9eebf84d..031cfdfe 100755 --- a/travis-build-scripts/test.sh +++ b/travis-build-scripts/test.sh @@ -17,7 +17,7 @@ set -e # Use verbose test output -TEST_OPTS_DOCKER="-v" +export TEST_OPTS_DOCKER="-v" echo 'Testing Developer image...' && echo -en 'travis_fold:start:test-devserver\\r' make test-devserver