[ci skip]: Update v9.3.1 branch for MQ 9.3.1.0-r2 (#350)

* Update MQ version to 9.3.1.0-r2

* First part of the changes for SSLKEYR  (#328)

* Squashed all commits

* Addressed review comments

* Fix JMS test build issue (#340)

* Fix JMS test build issue

* Remove ciphername where not required

* Fix issue1766 and add test case (#336)

* Fix issue1766 and add test case

* Address review comments

* Updated copyright year

* Resolve merge conflicts

* Updating changelog (#346)

* Updating changelog

* Updating changelog

* updated go-version & ubi

.travis.yml

@@ -27,7 +27,7 @@ env:
   global:
     - MAIN_BRANCH=v9.3.1
     - TAGCACHE_FILE=tagcache
-    - RELEASE=r1
+    - RELEASE=r2
 
 go_import_path: "github.com/ibm-messaging/mq-container"
 

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change log
 
+## 9.3.1.0-r2 (2022-11)
+
+* Queue manager attribute SSLKEYR is now set to blank instead of '/run/runmqserver/tls/key' if key and certificate are not supplied.
+
 ## 9.3.1.0 (2022-10)
 
 * Updated to MQ version 9.3.1.0

Dockerfile-server

@@ -13,9 +13,9 @@
 # limitations under the License.
 
 ARG BASE_IMAGE=registry.access.redhat.com/ubi8/ubi-minimal
-ARG BASE_TAG=8.6-941
+ARG BASE_TAG=8.7-923
 ARG BUILDER_IMAGE=registry.access.redhat.com/ubi8/go-toolset
-ARG BUILDER_TAG=1.17.12-7
+ARG BUILDER_TAG=1.17.12-11
 ARG GO_WORKDIR=/opt/app-root/src/go/src/github.com/ibm-messaging/mq-container
 ARG MQ_URL="https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/9.3.1.0-IBM-MQ-Advanced-for-Developers-Non-Install-LinuxX64.tar.gz"
 ###############################################################################

Makefile

@@ -304,7 +304,7 @@ build-devjmstest:
 test-devserver: test/docker/vendor
 	$(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER):$(MQ_TAG) on $(shell $(COMMAND) --version)"$(END)))
 	$(COMMAND) inspect $(MQ_IMAGE_DEVSERVER):$(MQ_TAG)
-	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=true DOCKER_API_VERSION=$(DOCKER_API_VERSION) go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) -tags mqdev $(TEST_OPTS_DOCKER)
+	cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER):$(MQ_TAG) EXPECTED_LICENSE=Developer DEV_JMS_IMAGE=$(DEV_JMS_IMAGE) IBMJRE=false DOCKER_API_VERSION=$(DOCKER_API_VERSION) go test -parallel $(NUM_CPU) -timeout $(TEST_TIMEOUT_DOCKER) -tags mqdev $(TEST_OPTS_DOCKER)
 
 .PHONY: coverage
 coverage:

internal/keystore/keystore.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2018, 2020
+© Copyright IBM Corporation 2018, 2022
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -171,8 +171,8 @@ func (ks *KeyStore) GetCertificateLabels() ([]string, error) {
 	var labels []string
 	for scanner.Scan() {
 		s := scanner.Text()
-		if strings.HasPrefix(s, "-") || strings.HasPrefix(s, "*-") {
-			s := strings.TrimLeft(s, "-*")
+		if strings.HasPrefix(s, "-") || strings.HasPrefix(s, "*-") || strings.HasPrefix(s, "!") {
+			s := strings.TrimLeft(s, "-*!")
 			labels = append(labels, strings.TrimSpace(s))
 		}
 	}

internal/tls/tls.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019, 2021
+© Copyright IBM Corporation 2019, 2022
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -76,18 +76,20 @@ type TLSStore struct {
 	Truststore KeyStoreData
 }
 
-func configureTLSKeystores(keystoreDir, keyDir, trustDir string, p12TruststoreRequired bool) (string, KeyStoreData, KeyStoreData, error) {
-
+func configureTLSKeystores(keystoreDir, keyDir, trustDir string, p12TruststoreRequired bool, nativeTLSHA bool) (string, KeyStoreData, KeyStoreData, error) {
+	var keyLabel string
 	// Create the CMS Keystore & PKCS#12 Truststore (if required)
-	tlsStore, err := generateAllKeystores(keystoreDir, p12TruststoreRequired)
+	tlsStore, err := generateAllKeystores(keystoreDir, p12TruststoreRequired, nativeTLSHA)
 	if err != nil {
 		return "", tlsStore.Keystore, tlsStore.Truststore, err
 	}
 
-	// Process all keys - add them to the CMS KeyStore
-	keyLabel, err := processKeys(&tlsStore, keystoreDir, keyDir)
-	if err != nil {
-		return "", tlsStore.Keystore, tlsStore.Truststore, err
+	if tlsStore.Keystore.Keystore != nil {
+		// Process all keys - add them to the CMS KeyStore
+		keyLabel, err = processKeys(&tlsStore, keystoreDir, keyDir)
+		if err != nil {
+			return "", tlsStore.Keystore, tlsStore.Truststore, err
+		}
 	}
 
 	// Process all trust certificates - add them to the CMS KeyStore & PKCS#12 Truststore (if required)
@@ -101,23 +103,32 @@ func configureTLSKeystores(keystoreDir, keyDir, trustDir string, p12TruststoreRe
 
 // ConfigureDefaultTLSKeystores configures the CMS Keystore & PKCS#12 Truststore
 func ConfigureDefaultTLSKeystores() (string, KeyStoreData, KeyStoreData, error) {
-	return configureTLSKeystores(keystoreDirDefault, keyDirDefault, trustDirDefault, true)
+	return configureTLSKeystores(keystoreDirDefault, keyDirDefault, trustDirDefault, true, false)
 }
 
 // ConfigureHATLSKeystore configures the CMS Keystore & PKCS#12 Truststore
 func ConfigureHATLSKeystore() (string, KeyStoreData, KeyStoreData, error) {
 	// *.crt files mounted to the HA TLS dir keyDirHA will be processed as trusted in the CMS keystore
-	return configureTLSKeystores(keystoreDirHA, keyDirHA, keyDirHA, false)
+	return configureTLSKeystores(keystoreDirHA, keyDirHA, keyDirHA, false, true)
 }
 
 // ConfigureTLS configures TLS for the queue manager
 func ConfigureTLS(keyLabel string, cmsKeystore KeyStoreData, devMode bool, log *logger.Logger) error {
 
 	const mqsc string = "/etc/mqm/15-tls.mqsc"
 	const mqscTemplate string = mqsc + ".tpl"
-
+	sslKeyRing := ""
+
+	// Don't set SSLKEYR if no keys or crts are not supplied
+	// Key label will be blank if no certs were added during processing keys and certs.
+	if cmsKeystore.Keystore != nil {
+		certList, _ := cmsKeystore.Keystore.ListAllCertificates()
+		if len(certList) > 0 {
+			sslKeyRing = strings.TrimSuffix(cmsKeystore.Keystore.Filename, ".kdb")
+		}
+	}
 	err := mqtemplate.ProcessTemplateFile(mqscTemplate, mqsc, map[string]string{
-		"SSLKeyR":          strings.TrimSuffix(cmsKeystore.Keystore.Filename, ".kdb"),
+		"SSLKeyR":          sslKeyRing,
 		"CertificateLabel": keyLabel,
 	}, log)
 	if err != nil {
@@ -159,7 +170,7 @@ func configureTLSDev(log *logger.Logger) error {
 }
 
 // generateAllKeystores creates the CMS Keystore & PKCS#12 Truststore (if required)
-func generateAllKeystores(keystoreDir string, p12TruststoreRequired bool) (TLSStore, error) {
+func generateAllKeystores(keystoreDir string, p12TruststoreRequired bool, nativeTLSHA bool) (TLSStore, error) {
 
 	var cmsKeystore, p12Truststore KeyStoreData
 
@@ -175,11 +186,19 @@ func generateAllKeystores(keystoreDir string, p12TruststoreRequired bool) (TLSSt
 		return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create Keystore directory: %v", err)
 	}
 
-	// Create the CMS Keystore
-	cmsKeystore.Keystore = keystore.NewCMSKeyStore(filepath.Join(keystoreDir, cmsKeystoreName), cmsKeystore.Password)
-	err = cmsKeystore.Keystore.Create()
-	if err != nil {
-		return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create CMS Keystore: %v", err)
+	// Search the default keys directory for any keys/certs.
+	keysDirectory := keyDirDefault
+	// Change to default native HA TLS directory if we are configuring nativeHA
+	if nativeTLSHA {
+		keysDirectory = keyDirHA
+	}
+	// Create the CMS Keystore if we have been provided keys and certificates
+	if haveKeysAndCerts(keysDirectory) || haveKeysAndCerts(trustDirDefault) {
+		cmsKeystore.Keystore = keystore.NewCMSKeyStore(filepath.Join(keystoreDir, cmsKeystoreName), cmsKeystore.Password)
+		err = cmsKeystore.Keystore.Create()
+		if err != nil {
+			return TLSStore{cmsKeystore, p12Truststore}, fmt.Errorf("Failed to create CMS Keystore: %v", err)
+		}
 	}
 
 	// Create the PKCS#12 Truststore (if required)
@@ -203,7 +222,6 @@ func processKeys(tlsStore *TLSStore, keystoreDir string, keyDir string) (string,
 	// Process all keys
 	keyList, err := ioutil.ReadDir(keyDir)
 	if err == nil && len(keyList) > 0 {
-
 		// Process each set of keys - each set should contain files: *.key & *.crt
 		for _, keySet := range keyList {
 			keys, _ := ioutil.ReadDir(filepath.Join(keyDir, keySet.Name()))
@@ -602,3 +620,23 @@ func writeCertificatesToFile(file string, certificates []*pem.Block) error {
 	}
 	return nil
 }
+
+// Search the specified directory for .key and .crt files.
+// Return true if at least one .key or .crt file is found else false
+func haveKeysAndCerts(keyDir string) bool {
+	fileList, err := os.ReadDir(keyDir)
+	if err == nil && len(fileList) > 0 {
+		for _, fileInfo := range fileList {
+			// Keys and certs will be supplied in an user defined subdirectory.
+			// Do a listing of the subdirectory and then search for .key and .cert files
+			keys, _ := ioutil.ReadDir(filepath.Join(keyDir, fileInfo.Name()))
+			for _, key := range keys {
+				if strings.Contains(key.Name(), ".key") || strings.Contains(key.Name(), ".crt") {
+					// We found at least one key/crt file.
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

internal/tls/tls_web.go

@@ -65,7 +65,6 @@ func ConfigureWebKeystore(p12Truststore KeyStoreData, webKeystore string) (strin
 	// Check if a new self-signed certificate should be generated
 	genHostName := os.Getenv("MQ_GENERATE_CERTIFICATE_HOSTNAME")
 	if genHostName != "" {
-
 		// Create the Web Keystore
 		newWebKeystore := keystore.NewPKCS12KeyStore(webKeystoreFile, p12Truststore.Password)
 		err := newWebKeystore.Create()

test/docker/devconfig_test.go

@@ -1,3 +1,4 @@
+//go:build mqdev
 // +build mqdev
 
 /*
@@ -51,8 +52,10 @@ func TestDevGoldenPath(t *testing.T) {
 	waitForReady(t, cli, id)
 	waitForWebReady(t, cli, id, insecureTLSConfig)
 	t.Run("JMS", func(t *testing.T) {
-		// Run the JMS tests, with no password specified
-		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS)
+		// Run the JMS tests, with no password specified.
+		// Use OpenJDK JRE for running testing, pass false for 7th parameter.
+		// Last parameter is blank as the test doesn't use TLS.
+		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS, "false", "")
 	})
 	t.Run("REST admin", func(t *testing.T) {
 		testRESTAdmin(t, cli, id, insecureTLSConfig)
@@ -115,7 +118,9 @@ func TestDevSecure(t *testing.T) {
 	waitForWebReady(t, cli, ctr.ID, createTLSConfig(t, cert, tlsPassPhrase))
 
 	t.Run("JMS", func(t *testing.T) {
-		runJMSTests(t, cli, ctr.ID, true, "app", appPassword)
+		// OpenJDK is used for running tests, hence pass "false" for 7th parameter.
+		// Cipher name specified is compliant with non-IBM JRE naming.
+		runJMSTests(t, cli, ctr.ID, true, "app", appPassword, "false", "TLS_RSA_WITH_AES_256_CBC_SHA256")
 	})
 	t.Run("REST admin", func(t *testing.T) {
 		testRESTAdmin(t, cli, ctr.ID, insecureTLSConfig)
@@ -153,7 +158,9 @@ func TestDevWebDisabled(t *testing.T) {
 	})
 	t.Run("JMS", func(t *testing.T) {
 		// Run the JMS tests, with no password specified
-		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS)
+		// OpenJDK is used for running tests, hence pass "false" for 7th parameter.
+		// Last parameter is blank as the test doesn't use TLS.
+		runJMSTests(t, cli, id, false, "app", defaultAppPasswordOS, "false", "")
 	})
 	// Stop the container cleanly
 	stopContainer(t, cli, id)
@@ -184,3 +191,131 @@ func TestDevConfigDisabled(t *testing.T) {
 	// Stop the container cleanly
 	stopContainer(t, cli, id)
 }
+
+// Test if SSLKEYR and CERTLABL attributes are not set when key and certificate
+// are not supplied.
+func TestSSLKEYRBlank(t *testing.T) {
+	t.Parallel()
+
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=qm1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+		},
+	}
+	id := runContainerWithPorts(t, cli, &containerConfig, []int{9443})
+	defer cleanContainer(t, cli, id)
+	waitForReady(t, cli, id)
+	// execute runmqsc to display qmgr SSLKEYR and CERTLABL attibutes.
+	// Search the console output for exepcted values
+	_, sslkeyROutput := execContainer(t, cli, id, "", []string{"bash", "-c", "echo 'DISPLAY QMGR SSLKEYR CERTLABL' | runmqsc"})
+	if !strings.Contains(sslkeyROutput, "SSLKEYR( )") && !strings.Contains(sslkeyROutput, "CERTLABL( )") {
+		t.Errorf("Expected SSLKEYR to be blank but it is not; got \"%v\"", sslkeyROutput)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, id)
+}
+
+// Test if SSLKEYR and CERTLABL attributes are set when key and certificate
+// are supplied.
+func TestSSLKEYRWithSuppliedKeyAndCert(t *testing.T) {
+	t.Parallel()
+
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=QM1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+		},
+		Image: imageName(),
+	}
+	hostConfig := container.HostConfig{
+		Binds: []string{
+			coverageBind(t),
+			tlsDir(t, false) + ":/etc/mqm/pki/keys/default",
+		},
+	}
+	networkingConfig := network.NetworkingConfig{}
+	ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanContainer(t, cli, ctr.ID)
+	startContainer(t, cli, ctr.ID)
+	waitForReady(t, cli, ctr.ID)
+	// execute runmqsc to display qmgr SSLKEYR and CERTLABL attibutes.
+	// Search the console output for exepcted values
+	_, sslkeyROutput := execContainer(t, cli, ctr.ID, "", []string{"bash", "-c", "echo 'DISPLAY QMGR SSLKEYR CERTLABL' | runmqsc"})
+	if !strings.Contains(sslkeyROutput, "SSLKEYR(/run/runmqserver/tls/key)") && !strings.Contains(sslkeyROutput, "CERTLABL(default)") {
+		t.Errorf("Expected SSLKEYR to be '/run/runmqserver/tls/key' but it is not; got \"%v\"", sslkeyROutput)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, ctr.ID)
+}
+
+// Test with CA cert
+func TestSSLKEYRWithCACert(t *testing.T) {
+	t.Parallel()
+
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	containerConfig := container.Config{
+		Env: []string{
+			"LICENSE=accept",
+			"MQ_QMGR_NAME=QM1",
+			"MQ_ENABLE_EMBEDDED_WEB_SERVER=false",
+		},
+		Image: imageName(),
+	}
+	hostConfig := container.HostConfig{
+		Binds: []string{
+			coverageBind(t),
+			tlsDirWithCA(t, false) + ":/etc/mqm/pki/keys/QM1CA",
+		},
+		// Assign a random port for the web server on the host
+		PortBindings: nat.PortMap{
+			"9443/tcp": []nat.PortBinding{
+				{
+					HostIP: "0.0.0.0",
+				},
+			},
+		},
+	}
+	networkingConfig := network.NetworkingConfig{}
+	ctr, err := cli.ContainerCreate(context.Background(), &containerConfig, &hostConfig, &networkingConfig, t.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cleanContainer(t, cli, ctr.ID)
+	startContainer(t, cli, ctr.ID)
+	waitForReady(t, cli, ctr.ID)
+
+	// execute runmqsc to display qmgr SSLKEYR and CERTLABL attibutes.
+	// Search the console output for exepcted values
+	_, sslkeyROutput := execContainer(t, cli, ctr.ID, "", []string{"bash", "-c", "echo 'DISPLAY QMGR SSLKEYR CERTLABL' | runmqsc"})
+	if !strings.Contains(sslkeyROutput, "SSLKEYR(/run/runmqserver/tls/key)") {
+		t.Errorf("Expected SSLKEYR to be '/run/runmqserver/tls/key' but it is not; got \"%v\"", sslkeyROutput)
+	}
+
+	if !strings.Contains(sslkeyROutput, "CERTLABL(QM1CA)") {
+		t.Errorf("Expected CERTLABL to be 'QM1CA' but it is not; got \"%v\"", sslkeyROutput)
+	}
+
+	// Stop the container cleanly
+	stopContainer(t, cli, ctr.ID)
+}