From 4403c18d0f6cbfd93a90fa7f721bccf432842cbc Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Mon, 11 Mar 2019 15:56:50 +0000 Subject: [PATCH 1/5] fix for large MQSC scripts failing --- cmd/runmqserver/qmgr.go | 49 +++++++++++++--------------------- test/docker/docker_api_test.go | 43 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 31 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index 49b73c19..deeda3c7 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -16,6 +16,7 @@ limitations under the License. package main import ( + "bytes" "io" "io/ioutil" "os" @@ -86,43 +87,29 @@ func configureQueueManager() error { log.Println(err) return err } - for _, file := range files { if strings.HasSuffix(file.Name(), ".mqsc") { abs := filepath.Join(configDir, file.Name()) // #nosec G204 - cmd := exec.Command("runmqsc") - stdin, err := cmd.StdinPipe() - if err != nil { - log.Println(err) - return err - } - // Open the MQSC file for reading - // #nosec G304 - f, err := os.Open(abs) - if err != nil { - log.Printf("Error opening %v: %v", abs, err) - } - // Copy the contents to stdin of the runmqsc process - _, err = io.Copy(stdin, f) - if err != nil { - log.Errorf("Error reading %v: %v", abs, err) - } - err = f.Close() - if err != nil { - log.Errorf("Failed to close MQSC file handle: %v", err) - } - err = stdin.Close() - if err != nil { - log.Errorf("Failed to close MQSC stdin: %v", err) - } - // Run the command and wait for completion - out, err := cmd.CombinedOutput() + cmd1 := exec.Command("cat", abs) + cmd2 := exec.Command("runmqsc") + reader, writer := io.Pipe() + defer writer.Close() + defer reader.Close() + cmd1.Stdout = writer + cmd2.Stdin = reader + var buffer2 bytes.Buffer + cmd2.Stdout = &buffer2 + cmd1.Start() + cmd2.Start() + cmd1.Wait() + cmd2.Wait() + out := os.Stdout + _, err := io.Copy(out, &buffer2) if err != nil { - log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) + log.Error("Error running MQSC file %v :\n\t%v", file.Name(), err) } - // Print the runmqsc output, adding tab characters to make it more readable as part of the log - log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) + log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, out) } } return nil diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index a761112d..db0f2df3 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -534,6 +534,49 @@ func TestMQSC(t *testing.T) { } } +// TestLargeMQSC creates a new image with a large MQSC file in, starts a container based +// on that image, and checks that the MQSC has been applied correctly. +func TestLargeMQSC(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + const numQueues = 1000 + var buf bytes.Buffer + for i := 1; i <= numQueues; i++ { + fmt.Fprintf(&buf, "* Test processing of a large MQSC file, defining queue test%v\nDEFINE QLOCAL(test%v)\n", i, i) + } + var files = []struct { + Name, Body string + }{ + {"Dockerfile", fmt.Sprintf(` + FROM %v + USER root + RUN rm -f /etc/mqm/*.mqsc + ADD test.mqsc /etc/mqm/ + RUN chmod 0660 /etc/mqm/test.mqsc + USER mqm`, imageName())}, + {"test.mqsc", buf.String()}, + } + tag := createImage(t, cli, files) + defer deleteImage(t, cli, tag) + + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + Image: tag, + } + id := runContainer(t, cli, &containerConfig) + // defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + + rc, mqscOutput := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test" + strconv.Itoa(numQueues) + ")' | runmqsc"}) + if rc != 0 { + r := regexp.MustCompile("AMQ[0-9][0-9][0-9][0-9]E") + t.Fatalf("Expected runmqsc to exit with rc=0, got %v with error %v", rc, r.FindString(mqscOutput)) + } +} + // TestInvalidMQSC creates a new image with an MQSC file containing invalid MQSC, // tries to start a container based on that image, and checks that container terminates // func TestInvalidMQSC(t *testing.T) { From 37b7dd7c686526f5ffb9345e8720c39cc89cc8f9 Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Mon, 11 Mar 2019 17:06:45 +0000 Subject: [PATCH 2/5] defer reader close only --- cmd/runmqserver/qmgr.go | 2 +- test/docker/docker_api_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index deeda3c7..a726e44b 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -94,7 +94,6 @@ func configureQueueManager() error { cmd1 := exec.Command("cat", abs) cmd2 := exec.Command("runmqsc") reader, writer := io.Pipe() - defer writer.Close() defer reader.Close() cmd1.Stdout = writer cmd2.Stdin = reader @@ -103,6 +102,7 @@ func configureQueueManager() error { cmd1.Start() cmd2.Start() cmd1.Wait() + writer.Close() cmd2.Wait() out := os.Stdout _, err := io.Copy(out, &buffer2) diff --git a/test/docker/docker_api_test.go b/test/docker/docker_api_test.go index db0f2df3..1d9b525d 100644 --- a/test/docker/docker_api_test.go +++ b/test/docker/docker_api_test.go @@ -567,7 +567,7 @@ func TestLargeMQSC(t *testing.T) { Image: tag, } id := runContainer(t, cli, &containerConfig) - // defer cleanContainer(t, cli, id) + defer cleanContainer(t, cli, id) waitForReady(t, cli, id) rc, mqscOutput := execContainer(t, cli, id, "mqm", []string{"bash", "-c", "echo 'DISPLAY QLOCAL(test" + strconv.Itoa(numQueues) + ")' | runmqsc"}) From febadd3d043f8eb003deedad69c380274252ec09 Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Tue, 12 Mar 2019 14:19:06 +0000 Subject: [PATCH 3/5] tidy up --- cmd/runmqserver/qmgr.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index a726e44b..a44c7616 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -90,7 +90,7 @@ func configureQueueManager() error { for _, file := range files { if strings.HasSuffix(file.Name(), ".mqsc") { abs := filepath.Join(configDir, file.Name()) - // #nosec G204 + // Pipe the mqsc file into runmqsc cmd1 := exec.Command("cat", abs) cmd2 := exec.Command("runmqsc") reader, writer := io.Pipe() @@ -99,6 +99,7 @@ func configureQueueManager() error { cmd2.Stdin = reader var buffer2 bytes.Buffer cmd2.Stdout = &buffer2 + // Run the command and wait for completion cmd1.Start() cmd2.Start() cmd1.Wait() From d4014a5f6fc043b15800aa5c818f92164b01acf5 Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Thu, 14 Mar 2019 19:11:25 +0000 Subject: [PATCH 4/5] format stdout --- cmd/runmqserver/qmgr.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index a44c7616..d07fa12c 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -104,13 +104,13 @@ func configureQueueManager() error { cmd2.Start() cmd1.Wait() writer.Close() - cmd2.Wait() - out := os.Stdout - _, err := io.Copy(out, &buffer2) + err := cmd2.Wait() + out := buffer2.String() + // _, err := io.Copy(out, &buffer2) if err != nil { - log.Error("Error running MQSC file %v :\n\t%v", file.Name(), err) + log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) } - log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, out) + log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1)) } } return nil From 6d291944be3001d4cd9398dd57c443783703005e Mon Sep 17 00:00:00 2001 From: Luke Powlett Date: Thu, 14 Mar 2019 19:13:41 +0000 Subject: [PATCH 5/5] tidy up --- cmd/runmqserver/qmgr.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/runmqserver/qmgr.go b/cmd/runmqserver/qmgr.go index d07fa12c..8dca3f5a 100644 --- a/cmd/runmqserver/qmgr.go +++ b/cmd/runmqserver/qmgr.go @@ -106,7 +106,6 @@ func configureQueueManager() error { writer.Close() err := cmd2.Wait() out := buffer2.String() - // _, err := io.Copy(out, &buffer2) if err != nil { log.Errorf("Error running MQSC file %v (%v):\n\t%v", file.Name(), err, strings.Replace(string(out), "\n", "\n\t", -1)) }