From 10e448056bcbdad275415b7519d6077a37c6ec9c Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 12 Mar 2018 11:14:59 +0000 Subject: [PATCH 1/5] Add web server to dev image --- Makefile | 12 +- cmd/runmqdevserver/main.go | 133 ++++++++++++++++++ cmd/runmqdevserver/mqsc.go | 57 ++++++++ cmd/runmqserver/main.go | 6 + cmd/runmqserver/mirror.go | 4 +- cmd/runmqserver/post_init_dev.go | 36 +++++ cmd/runmqserver/post_init_other.go | 22 +++ cmd/runmqserver/webserver.go | 110 +++++++++++++++ docs/building.md | 2 + incubating/mqadvanced-server-dev/Dockerfile | 40 +++++- .../{dev.mqsc => dev.mqsc.tpl} | 4 +- .../Installation1/servers/mqweb/mqwebuser.xml | 30 ++++ .../Installation1/servers/mqweb/tls-dev.xml | 4 + .../Installation1/servers/mqweb/tls.xml | 1 + install-mq.sh | 2 +- internal/command/command.go | 46 +++++- test/docker/devconfig_test.go | 72 ++++++++++ test/docker/devconfig_test_util.go | 42 ++++++ test/docker/docker_api_test_util.go | 18 +++ 19 files changed, 621 insertions(+), 20 deletions(-) create mode 100644 cmd/runmqdevserver/main.go create mode 100644 cmd/runmqdevserver/mqsc.go create mode 100644 cmd/runmqserver/post_init_dev.go create mode 100644 cmd/runmqserver/post_init_other.go create mode 100644 cmd/runmqserver/webserver.go rename incubating/mqadvanced-server-dev/{dev.mqsc => dev.mqsc.tpl} (94%) create mode 100644 incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml create mode 100644 incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml create mode 100644 incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml create mode 100644 test/docker/devconfig_test.go create mode 100644 test/docker/devconfig_test_util.go diff --git a/Makefile b/Makefile index daeed668..50edd0d7 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,9 @@ MQ_VERSION ?= 9.0.4.0 # be installed. The default value is derived from MQ_VERSION, BASE_IMAGE and architecture # Does not apply to MQ Advanced for Developers. MQ_ARCHIVE ?= IBM_MQ_$(MQ_VERSION)_$(MQ_ARCHIVE_TYPE)_$(MQ_ARCHIVE_ARCH).tar.gz +# MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced +# for Developers can be installed +MQ_ARCHIVE_DEV ?= $(MQ_ARCHIVE_DEV_$(MQ_VERSION)) # Options to `go test` for the Docker tests TEST_OPTS_DOCKER ?= # Options to `go test` for the Kubernetes tests @@ -68,9 +71,6 @@ endif # Archive names for IBM MQ Advanced for Developers for Ubuntu MQ_ARCHIVE_DEV_9.0.3.0=mqadv_dev903_ubuntu_x86-64.tar.gz MQ_ARCHIVE_DEV_9.0.4.0=mqadv_dev904_ubuntu_x86-64.tar.gz -# MQ_ARCHIVE_DEV is the name of the file, under the downloads directory, from which MQ Advanced -# for Developers can be installed -MQ_ARCHIVE_DEV=$(MQ_ARCHIVE_DEV_$(MQ_VERSION)) ############################################################################### # Build targets @@ -140,7 +140,7 @@ test-advancedserver: test/docker/vendor .PHONY: test-devserver test-devserver: test/docker/vendor $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) - cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) + cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) .PHONY: test-advancedserver-cover test-advancedserver-cover: test/docker/vendor @@ -218,11 +218,13 @@ build-advancedserver: downloads/$(MQ_ARCHIVE) docker-version $(call docker-build-mq,$(MQ_IMAGE_ADVANCEDSERVER),Dockerfile-server,$(MQ_ARCHIVE),"4486e8c4cc9146fd9b3ce1f14a2dfc5b","IBM MQ Advanced",$(MQ_VERSION)) .PHONY: build-devserver +# Target-specific variable to add web server into devserver image +build-devserver: MQ_PACKAGES=ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams ibmmq-web build-devserver: downloads/$(MQ_ARCHIVE_DEV) docker-version @test "$(shell uname -m)" = "x86_64" || (echo "Error: MQ Advanced for Developers is only available for x86_64 architecture" && exit 1) $(info $(shell printf $(TITLE)"Build $(MQ_IMAGE_DEVSERVER_BASE)"$(END))) $(call docker-build-mq,$(MQ_IMAGE_DEVSERVER_BASE),Dockerfile-server,$(MQ_ARCHIVE_DEV),"98102d16795c4263ad9ca075190a2d4d","IBM MQ Advanced for Developers (Non-Warranted)",$(MQ_VERSION)) - docker build --tag $(MQ_IMAGE_DEVSERVER) incubating/mqadvanced-server-dev + docker build --tag $(MQ_IMAGE_DEVSERVER) --file incubating/mqadvanced-server-dev/Dockerfile . .PHONY: build-advancedserver-cover build-advancedserver-cover: docker-version diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go new file mode 100644 index 00000000..d1607aa7 --- /dev/null +++ b/cmd/runmqdevserver/main.go @@ -0,0 +1,133 @@ +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "syscall" + + "github.com/ibm-messaging/mq-container/internal/command" + "github.com/ibm-messaging/mq-container/internal/logger" +) + +var log *logger.Logger + +func setPassword(user string, password string) error { + cmd := exec.Command("chpasswd") + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + fmt.Fprintf(stdin, "%s:%s", user, password) + stdin.Close() + _, _, err = command.RunCmd(cmd) + if err != nil { + return err + } + log.Printf("Set password for \"%v\" user", user) + return nil +} + +func getLogFormat() string { + return os.Getenv("LOG_FORMAT") +} + +func getDebug() bool { + debug := os.Getenv("DEBUG") + if debug == "true" || debug == "1" { + return true + } + return false +} + +func configureLogger() error { + f := getLogFormat() + d := getDebug() + switch f { + case "json": + log = logger.NewLogger(os.Stderr, d, true) + case "basic": + log = logger.NewLogger(os.Stderr, d, false) + default: + log = logger.NewLogger(os.Stdout, d, false) + return fmt.Errorf("invalid value for LOG_FORMAT: %v", f) + } + return nil +} + +func logTerminationf(format string, args ...interface{}) { + logTermination(fmt.Sprintf(format, args)) +} + +// TODO: Duplicated code +func logTermination(args ...interface{}) { + msg := fmt.Sprint(args) + // Write the message to the termination log. This is the default place + // that Kubernetes will look for termination information. + log.Debugf("Writing termination message: %v", msg) + err := ioutil.WriteFile("/dev/termination-log", []byte(msg), 0660) + if err != nil { + log.Debug(err) + } + log.Error(msg) +} + +func doMain() error { + err := configureLogger() + if err != nil { + logTermination(err) + return err + } + adminPassword, set := os.LookupEnv("MQ_ADMIN_PASSWORD") + if set { + err = setPassword("admin", adminPassword) + if err != nil { + logTerminationf("Error setting admin password: %v", err) + return err + } + } + appPassword, set := os.LookupEnv("MQ_APP_PASSWORD") + if set { + err = setPassword("app", appPassword) + if err != nil { + logTerminationf("Error setting app password: %v", err) + return err + } + } + + err = updateMQSC(set) + if err != nil { + logTerminationf("Error updating MQSC: %v", err) + return err + } + + return nil +} + +var osExit = os.Exit + +func main() { + err := doMain() + if err != nil { + osExit(1) + } else { + // Replace this process with runmqserver + syscall.Exec("/usr/local/bin/runmqserver", []string{"runmqserver"}, os.Environ()) + } +} diff --git a/cmd/runmqdevserver/mqsc.go b/cmd/runmqdevserver/mqsc.go new file mode 100644 index 00000000..a30e2671 --- /dev/null +++ b/cmd/runmqdevserver/mqsc.go @@ -0,0 +1,57 @@ +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "html/template" + "os" +) + +func updateMQSC(appPasswordRequired bool) error { + var checkClient string + if appPasswordRequired { + checkClient = "REQUIRED" + } else { + checkClient = "ASQMGR" + } + const mqsc string = "/etc/mqm/dev.mqsc" + if os.Getenv("MQ_DEV") == "true" { + const mqscTemplate string = mqsc + ".tpl" + // Re-configure channel if app password not set + t, err := template.ParseFiles(mqscTemplate) + if err != nil { + log.Error(err) + return err + } + f, err := os.OpenFile(mqsc, os.O_CREATE|os.O_WRONLY, 0660) + defer f.Close() + err = t.Execute(f, map[string]string{"ChckClnt": checkClient}) + if err != nil { + log.Error(err) + return err + } + // TODO: Lookup value for MQM user here? + err = os.Chown(mqsc, 999, 999) + if err != nil { + log.Error(err) + return err + } + // os.Remove(mqscTemplate) + } else { + os.Remove(mqsc) + } + return nil +} diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index c24c4670..59ec0e23 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -68,6 +68,12 @@ func doMain() error { if err != nil { return err } + + err = postInit(name) + if err != nil { + return err + } + newQM, err := createQueueManager(name) if err != nil { logTermination(err) diff --git a/cmd/runmqserver/mirror.go b/cmd/runmqserver/mirror.go index 1e2f38c4..170e6082 100644 --- a/cmd/runmqserver/mirror.go +++ b/cmd/runmqserver/mirror.go @@ -61,7 +61,9 @@ func mirrorAvailableMessages(f *os.File, mf mirrorFunc) { mf(t) count++ } - log.Debugf("Mirrored %v log entries from %v", count, f.Name()) + if count > 0 { + log.Debugf("Mirrored %v log entries from %v", count, f.Name()) + } err := scanner.Err() if err != nil { log.Errorf("Error reading file %v: %v", f.Name(), err) diff --git a/cmd/runmqserver/post_init_dev.go b/cmd/runmqserver/post_init_dev.go new file mode 100644 index 00000000..ddbe7f1b --- /dev/null +++ b/cmd/runmqserver/post_init_dev.go @@ -0,0 +1,36 @@ +// +build mqdev + +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import "os" + +// postInit is run after /var/mqm is set up +// This version of postInit is only included as part of the MQ Advanced for Developers build +func postInit(name string) error { + disable := os.Getenv("MQ_DISABLE_WEB_CONSOLE") + if disable != "true" && disable != "1" { + // Configure and start the web server, in the background (if installed) + // WARNING: No error handling or health checking available for the web server, + // which is why it's limited to use with MQ Advanced for Developers only + go func() { + configureWebServer() + startWebServer() + }() + } + return nil +} diff --git a/cmd/runmqserver/post_init_other.go b/cmd/runmqserver/post_init_other.go new file mode 100644 index 00000000..71f34582 --- /dev/null +++ b/cmd/runmqserver/post_init_other.go @@ -0,0 +1,22 @@ +// +build !mqdev + +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +func postInit(name string) error { + return nil +} diff --git a/cmd/runmqserver/webserver.go b/cmd/runmqserver/webserver.go new file mode 100644 index 00000000..2224b450 --- /dev/null +++ b/cmd/runmqserver/webserver.go @@ -0,0 +1,110 @@ +// +build mqdev + +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/ibm-messaging/mq-container/internal/command" +) + +func startWebServer() error { + _, err := os.Stat("/opt/mqm/bin/strmqweb") + if err != nil && os.IsNotExist(err) { + log.Debug("Skipping web server, because it's not installed") + return nil + } + log.Println("Starting web server") + out, rc, err := command.RunAsMQM("strmqweb") + if err != nil { + log.Printf("Error %v starting web server: %v", rc, string(out)) + return err + } + log.Println("Started web server") + return nil +} + +func configureWebServer() error { + _, err := os.Stat("/opt/mqm/bin/strmqweb") + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + const webConfigDir string = "/etc/mqm/web" + _, err = os.Stat(webConfigDir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + uid, gid, err := lookupMQM() + if err != nil { + return err + } + const prefix string = "/etc/mqm/web" + err = filepath.Walk(prefix, func(from string, info os.FileInfo, err error) error { + if err != nil { + return err + } + to := fmt.Sprintf("/var/mqm/web%v", from[len(prefix):]) + exists := true + _, err = os.Stat(to) + if err != nil { + if os.IsNotExist(err) { + exists = false + } else { + return err + } + } + + if info.IsDir() { + if !exists { + err := os.MkdirAll(to, 0770) + if err != nil { + return err + } + } + log.Printf("Directory: %v --> %v", from, to) + } else { + if exists { + err := os.Remove(to) + if err != nil { + return err + } + } + // TODO: Permissions. Can't rely on them being set in Dockerfile + err := os.Link(from, to) + if err != nil { + log.Debug(err) + return err + } + log.Printf("File: %v", from) + } + err = os.Chown(to, uid, gid) + if err != nil { + return err + } + return nil + }) + return err +} diff --git a/docs/building.md b/docs/building.md index 82b93e3e..da4b616c 100644 --- a/docs/building.md +++ b/docs/building.md @@ -29,6 +29,8 @@ MQ_ARCHIVE=mq-1.2.3.4.tar.gz MQ_VERSION=1.2.3.4 make build-advancedserver ## Building a developer image Run `make build-devserver`, which will download the latest version of MQ Advanced for Developers from IBM developerWorks. This is currently only available on the `x86_64` architecture. +You can use the environment variable `MQ_ARCHIVE_DEV` to specify an alternative local file to install from (which must be in the `downloads` directory). + ## Building on a different base image By default, the MQ images use Ubuntu as the base layer. You can build using a Red Hat Enterprise Linux compatible base layer by setting the `BASE_IMAGE` environment variable. For example: diff --git a/incubating/mqadvanced-server-dev/Dockerfile b/incubating/mqadvanced-server-dev/Dockerfile index 1f64e5e3..bf0ef99c 100644 --- a/incubating/mqadvanced-server-dev/Dockerfile +++ b/incubating/mqadvanced-server-dev/Dockerfile @@ -12,19 +12,45 @@ # See the License for the specific language governing permissions and # limitations under the License. +############################################################################### +# Build stage to build Go code +############################################################################### +FROM golang:1.9 as builder +WORKDIR /go/src/github.com/ibm-messaging/mq-container/ +COPY cmd/ ./cmd +COPY internal/ ./internal +COPY vendor/ ./vendor +# Re-build runmqserver, with code tagged with 'mqdev' enabled +RUN go build --tags 'mqdev' ./cmd/runmqserver +RUN go build ./cmd/runmqdevserver/ +# Run all unit tests +RUN go test -v ./cmd/runmqdevserver/... + +############################################################################### +# Main build stage +############################################################################### FROM mqadvanced-server-dev-base:9.0.4.0-x86_64-ubuntu-16.04 +# Enable MQ developer default configuration +ENV MQ_DEV=true + +# Default administrator password +ENV MQ_ADMIN_PASSWORD=passw0rd + ## Add admin and app users, and set a default password for admin RUN useradd admin -G mqm \ && groupadd mqclient \ && useradd app -G mqclient,mqm \ - && echo admin:passw0rd | chpasswd + && echo admin:$MQ_ADMIN_PASSWORD | chpasswd -COPY dev.mqsc /etc/mqm/ -COPY entrypoint.sh /usr/local/bin/ -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqserver /usr/local/bin/ +COPY --from=builder /go/src/github.com/ibm-messaging/mq-container/runmqdevserver /usr/local/bin/ +# Copy template MQSC for default developer configuration +COPY incubating/mqadvanced-server-dev/dev.mqsc.tpl /etc/mqm/ +# Copy web XML files for default developer configuration +COPY incubating/mqadvanced-server-dev/web /etc/mqm/web +RUN chmod +x /usr/local/bin/runmq* -# Enable MQ developer default configuration -ENV MQ_DEV=true +EXPOSE 9443 -ENTRYPOINT ["entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["runmqdevserver"] \ No newline at end of file diff --git a/incubating/mqadvanced-server-dev/dev.mqsc b/incubating/mqadvanced-server-dev/dev.mqsc.tpl similarity index 94% rename from incubating/mqadvanced-server-dev/dev.mqsc rename to incubating/mqadvanced-server-dev/dev.mqsc.tpl index ee2892f2..3dc733fe 100644 --- a/incubating/mqadvanced-server-dev/dev.mqsc +++ b/incubating/mqadvanced-server-dev/dev.mqsc.tpl @@ -1,4 +1,4 @@ -* © Copyright IBM Corporation 2017 +* © Copyright IBM Corporation 2017, 2018 * * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,7 +38,7 @@ DEFINE CHANNEL('DEV.APP.SVRCONN') CHLTYPE(SVRCONN) REPLACE * Developer channel authentication rules SET CHLAUTH('*') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(NOACCESS) DESCR('Back-stop rule - Blocks everyone') ACTION(REPLACE) -SET CHLAUTH('DEV.APP.SVRCONN') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT(REQUIRED) DESCR('Allows connection via APP channel') ACTION(REPLACE) +SET CHLAUTH('DEV.APP.SVRCONN') TYPE(ADDRESSMAP) ADDRESS('*') USERSRC(CHANNEL) CHCKCLNT({{ .ChckClnt }}) DESCR('Allows connection via APP channel') ACTION(REPLACE) SET CHLAUTH('DEV.ADMIN.SVRCONN') TYPE(BLOCKUSER) USERLIST('nobody') DESCR('Allows admins on ADMIN channel') ACTION(REPLACE) SET CHLAUTH('DEV.ADMIN.SVRCONN') TYPE(USERMAP) CLNTUSER('admin') USERSRC(CHANNEL) DESCR('Allows admin user to connect via ADMIN channel') ACTION(REPLACE) diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml new file mode 100644 index 00000000..14943276 --- /dev/null +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/mqwebuser.xml @@ -0,0 +1,30 @@ + + + + appSecurity-2.0 + basicAuthenticationMQ-1.0 + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml new file mode 100644 index 00000000..970941ba --- /dev/null +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls-dev.xml @@ -0,0 +1,4 @@ + + + + diff --git a/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml new file mode 100644 index 00000000..395b67f5 --- /dev/null +++ b/incubating/mqadvanced-server-dev/web/installations/Installation1/servers/mqweb/tls.xml @@ -0,0 +1 @@ + diff --git a/install-mq.sh b/install-mq.sh index ad3bf7c3..430b463b 100644 --- a/install-mq.sh +++ b/install-mq.sh @@ -22,7 +22,7 @@ test -f /usr/bin/yum && RHEL=true || RHEL=false test -f /usr/bin/apt-get && UBUNTU=true || UBUNTU=false # If MQ_PACKAGES isn't specifically set, then choose a valid set of defaults -if [ -z $MQ_PACKAGES ]; then +if [ -z "$MQ_PACKAGES" ]; then $UBUNTU && MQ_PACKAGES="ibmmq-server ibmmq-java ibmmq-jre ibmmq-gskit ibmmq-msg-.* ibmmq-samples ibmmq-ams" $RHEL && MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm MQSeriesMsg*.rpm MQSeriesSamples*.rpm MQSeriesAMS-*.rpm" fi diff --git a/internal/command/command.go b/internal/command/command.go index 0804cf27..a999d6c7 100644 --- a/internal/command/command.go +++ b/internal/command/command.go @@ -20,16 +20,17 @@ package command import ( "fmt" "os/exec" + "os/user" "runtime" + "strconv" "syscall" ) -// Run runs an OS command. On Linux it waits for the command to +// RunCmd runs an OS command. On Linux it waits for the command to // complete and returns the exit status (return code). // Do not use this function to run shell built-ins (like "cd"), because // the error handling works differently -func Run(name string, arg ...string) (string, int, error) { - cmd := exec.Command(name, arg...) +func RunCmd(cmd *exec.Cmd) (string, int, error) { // Run the command and wait for completion out, err := cmd.CombinedOutput() if err != nil { @@ -39,10 +40,47 @@ func Run(name string, arg ...string) (string, int, error) { if ok && runtime.GOOS == "linux" { status, ok := exiterr.Sys().(syscall.WaitStatus) if ok { - return string(out), status.ExitStatus(), fmt.Errorf("%v: %v", name, err) + return string(out), status.ExitStatus(), fmt.Errorf("%v: %v", cmd.Path, err) } } return string(out), -1, err } return string(out), 0, nil } + +// Run runs an OS command. On Linux it waits for the command to +// complete and returns the exit status (return code). +// Do not use this function to run shell built-ins (like "cd"), because +// the error handling works differently +func Run(name string, arg ...string) (string, int, error) { + return RunCmd(exec.Command(name, arg...)) +} + +// RunAsMQM runs the specified command as the mqm user +func RunAsMQM(name string, arg ...string) (string, int, error) { + cmd := exec.Command(name, arg...) + cmd.SysProcAttr = &syscall.SysProcAttr{} + uid, gid, err := lookupMQM() + if err != nil { + return "", 0, err + } + cmd.SysProcAttr.Credential = &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid)} + return RunCmd(cmd) +} + +// TODO: Duplicated code +func lookupMQM() (int, int, error) { + mqm, err := user.Lookup("mqm") + if err != nil { + return -1, -1, err + } + mqmUID, err := strconv.Atoi(mqm.Uid) + if err != nil { + return -1, -1, err + } + mqmGID, err := strconv.Atoi(mqm.Gid) + if err != nil { + return -1, -1, err + } + return mqmUID, mqmGID, nil +} diff --git a/test/docker/devconfig_test.go b/test/docker/devconfig_test.go new file mode 100644 index 00000000..4c8ddd49 --- /dev/null +++ b/test/docker/devconfig_test.go @@ -0,0 +1,72 @@ +// +build mqdev + +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "crypto/tls" + "fmt" + "net/http" + "testing" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" +) + +func TestDevGoldenPath(t *testing.T) { + t.Parallel() + cli, err := client.NewEnvClient() + if err != nil { + t.Fatal(err) + } + containerConfig := container.Config{ + Env: []string{"LICENSE=accept", "MQ_QMGR_NAME=qm1"}, + } + id := runContainer(t, cli, &containerConfig) + + defer cleanContainer(t, cli, id) + waitForReady(t, cli, id) + waitForWebReady(t, cli, id) + + timeout := time.Duration(30 * time.Second) + // Disable TLS verification (server uses a self-signed certificate by default, + // so verification isn't useful anyway) + tr := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + httpClient := http.Client{ + Timeout: timeout, + Transport: tr, + } + + url := fmt.Sprintf("https://localhost:%s/ibmmq/rest/v1/admin/installation", getWebPort(t, cli, id)) + req, err := http.NewRequest("GET", url, nil) + req.SetBasicAuth("admin", "passw0rd") + resp, err := httpClient.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected HTTP status code %v from 'GET installation'; got %v", http.StatusOK, resp.StatusCode) + } + + // Stop the container cleanly + stopContainer(t, cli, id) +} diff --git a/test/docker/devconfig_test_util.go b/test/docker/devconfig_test_util.go new file mode 100644 index 00000000..e3051cbf --- /dev/null +++ b/test/docker/devconfig_test_util.go @@ -0,0 +1,42 @@ +// +build mqdev + +/* +© Copyright IBM Corporation 2018 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package main + +import ( + "crypto/tls" + "fmt" + "testing" + "time" + + "github.com/docker/docker/client" +) + +func waitForWebReady(t *testing.T, cli *client.Client, ID string) { + config := tls.Config{InsecureSkipVerify: true} + a := fmt.Sprintf("localhost:%s", getWebPort(t, cli, ID)) + for { + conn, err := tls.Dial("tcp", a, &config) + if err == nil { + conn.Close() + // Extra sleep to allow web apps to start + time.Sleep(3 * time.Second) + t.Log("MQ web server is ready") + return + } + } +} diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 0dd5092d..7c54b3ee 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -39,6 +39,7 @@ import ( "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/go-connections/nat" ) func imageName() string { @@ -166,6 +167,15 @@ func runContainer(t *testing.T, cli *client.Client, containerConfig *container.C coverageBind(t), terminationBind(t), }, + // Assign a random port for the web server on the host + // TODO: Don't do this for all tests + PortBindings: nat.PortMap{ + "9443/tcp": []nat.PortBinding{ + { + HostIP: "0.0.0.0", + }, + }, + }, } networkingConfig := network.NetworkingConfig{} t.Logf("Running container (%s)", containerConfig.Image) @@ -493,3 +503,11 @@ func copyFromContainer(t *testing.T, cli *client.Client, id string, file string) } return b } + +func getWebPort(t *testing.T, cli *client.Client, ID string) string { + i, err := cli.ContainerInspect(context.Background(), ID) + if err != nil { + t.Fatal(err) + } + return i.NetworkSettings.Ports["9443/tcp"][0].HostPort +} From 98c594c91e826340ddec5250b6e93589266d53a5 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 12 Mar 2018 11:19:55 +0000 Subject: [PATCH 2/5] Fix log message in Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 50edd0d7..f7f1e2b0 100644 --- a/Makefile +++ b/Makefile @@ -134,12 +134,12 @@ test-unit: .PHONY: test-advancedserver test-advancedserver: test/docker/vendor - $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_ADVANCEDSERVER) on Docker"$(END))) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_ADVANCEDSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_ADVANCEDSERVER) go test -parallel $(NUM_CPU) $(TEST_OPTS_DOCKER) .PHONY: test-devserver test-devserver: test/docker/vendor - $(info $(SPACER)$(shell printf $(TITLE)"Test $(DOCKER_FULL_DEVSERVER) on Docker"$(END))) + $(info $(SPACER)$(shell printf $(TITLE)"Test $(MQ_IMAGE_DEVSERVER) on Docker"$(END))) cd test/docker && TEST_IMAGE=$(MQ_IMAGE_DEVSERVER) go test -parallel $(NUM_CPU) -tags mqdev $(TEST_OPTS_DOCKER) .PHONY: test-advancedserver-cover From d495b3640ead743b11eb5d22d3133fe0f6aaba84 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 12 Mar 2018 16:27:57 +0000 Subject: [PATCH 3/5] Add extra fields to runmqserver JSON log --- cmd/runmqdevserver/main.go | 18 +++++++++++++++--- cmd/runmqserver/logging.go | 18 ++++++++++++++---- cmd/runmqserver/main.go | 8 ++++---- cmd/runmqserver/main_test.go | 2 +- internal/logger/logger.go | 23 +++++++++++++++++++++-- internal/logger/logger_test.go | 12 +++++++++--- 6 files changed, 64 insertions(+), 17 deletions(-) diff --git a/cmd/runmqdevserver/main.go b/cmd/runmqdevserver/main.go index d1607aa7..28189b33 100644 --- a/cmd/runmqdevserver/main.go +++ b/cmd/runmqdevserver/main.go @@ -24,6 +24,7 @@ import ( "github.com/ibm-messaging/mq-container/internal/command" "github.com/ibm-messaging/mq-container/internal/logger" + "github.com/ibm-messaging/mq-container/internal/name" ) var log *logger.Logger @@ -57,15 +58,26 @@ func getDebug() bool { } func configureLogger() error { + var err error f := getLogFormat() d := getDebug() + n, err := name.GetQueueManagerName() + if err != nil { + return err + } switch f { case "json": - log = logger.NewLogger(os.Stderr, d, true) + log, err = logger.NewLogger(os.Stderr, d, true, n) + if err != nil { + return err + } case "basic": - log = logger.NewLogger(os.Stderr, d, false) + log, err = logger.NewLogger(os.Stderr, d, false, n) + if err != nil { + return err + } default: - log = logger.NewLogger(os.Stdout, d, false) + log, err = logger.NewLogger(os.Stdout, d, false, n) return fmt.Errorf("invalid value for LOG_FORMAT: %v", f) } return nil diff --git a/cmd/runmqserver/logging.go b/cmd/runmqserver/logging.go index a50b1e7d..9538ab94 100644 --- a/cmd/runmqserver/logging.go +++ b/cmd/runmqserver/logging.go @@ -89,15 +89,22 @@ func getDebug() bool { return false } -func configureLogger() (mirrorFunc, error) { +func configureLogger(name string) (mirrorFunc, error) { + var err error f := getLogFormat() d := getDebug() switch f { case "json": - log = logger.NewLogger(os.Stderr, d, true) + log, err = logger.NewLogger(os.Stderr, d, true, name) + if err != nil { + return nil, err + } return log.LogDirect, nil case "basic": - log = logger.NewLogger(os.Stderr, d, false) + log, err = logger.NewLogger(os.Stderr, d, false, name) + if err != nil { + return nil, err + } return func(msg string) { // Parse the JSON message, and print a simplified version var obj map[string]interface{} @@ -105,7 +112,10 @@ func configureLogger() (mirrorFunc, error) { fmt.Printf(formatSimple(obj["ibm_datetime"].(string), obj["message"].(string))) }, nil default: - log = logger.NewLogger(os.Stdout, d, false) + log, err = logger.NewLogger(os.Stdout, d, false, name) + if err != nil { + return nil, err + } return nil, fmt.Errorf("invalid value for LOG_FORMAT: %v", f) } } diff --git a/cmd/runmqserver/main.go b/cmd/runmqserver/main.go index 59ec0e23..af1baffd 100644 --- a/cmd/runmqserver/main.go +++ b/cmd/runmqserver/main.go @@ -28,17 +28,17 @@ import ( ) func doMain() error { - mf, err := configureLogger() + name, nameErr := name.GetQueueManagerName() + mf, err := configureLogger(name) if err != nil { logTermination(err) return err } - err = ready.Clear() - if err != nil { + if nameErr != nil { logTermination(err) return err } - name, err := name.GetQueueManagerName() + err = ready.Clear() if err != nil { logTermination(err) return err diff --git a/cmd/runmqserver/main_test.go b/cmd/runmqserver/main_test.go index fe60e703..ed4d5526 100644 --- a/cmd/runmqserver/main_test.go +++ b/cmd/runmqserver/main_test.go @@ -31,7 +31,7 @@ const filename = "/var/coverage/exitCode" func init() { test = flag.Bool("test", false, "Set to true when running tests for coverage") - log = logger.NewLogger(os.Stdout, true, false) + log, _ = logger.NewLogger(os.Stdout, true, false, "test") } // Test started when the test binary is started. Only calls main. diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 2aa5f32b..f0a8ed19 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "os/user" "sync" "time" ) @@ -39,10 +40,21 @@ type Logger struct { json bool processName string pid int + serverName string + host string + user *user.User } // NewLogger creates a new logger -func NewLogger(writer io.Writer, debug bool, json bool) *Logger { +func NewLogger(writer io.Writer, debug bool, json bool, serverName string) (*Logger, error) { + hostname, err := os.Hostname() + if err != nil { + return nil, err + } + user, err := user.Current() + if err != nil { + return nil, err + } return &Logger{ mutex: sync.Mutex{}, writer: writer, @@ -50,7 +62,10 @@ func NewLogger(writer io.Writer, debug bool, json bool) *Logger { json: json, processName: os.Args[0], pid: os.Getpid(), - } + serverName: serverName, + host: hostname, + user: user, + }, nil } func (l *Logger) format(entry map[string]interface{}) (string, error) { @@ -72,8 +87,12 @@ func (l *Logger) log(level string, msg string) { "message": fmt.Sprint(msg), "ibm_datetime": t.Format(timestampFormat), "loglevel": level, + "host": l.host, + "ibm_serverName": l.serverName, "ibm_processName": l.processName, "ibm_processId": l.pid, + "ibm_userName": l.user.Username, + "type": "mq_log", } s, err := l.format(entry) l.mutex.Lock() diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go index cf5fde35..49d8e999 100644 --- a/internal/logger/logger_test.go +++ b/internal/logger/logger_test.go @@ -25,11 +25,14 @@ import ( func TestJSONLogger(t *testing.T) { buf := new(bytes.Buffer) - l := NewLogger(buf, true, true) + l, err := NewLogger(buf, true, true, t.Name()) + if err != nil { + t.Fatal(err) + } s := "Hello world" l.Print(s) var e map[string]interface{} - err := json.Unmarshal([]byte(buf.String()), &e) + err = json.Unmarshal([]byte(buf.String()), &e) if err != nil { t.Error(err) } @@ -40,7 +43,10 @@ func TestJSONLogger(t *testing.T) { func TestSimpleLogger(t *testing.T) { buf := new(bytes.Buffer) - l := NewLogger(buf, true, false) + l, err := NewLogger(buf, true, false, t.Name()) + if err != nil { + t.Fatal(err) + } s := "Hello world" l.Print(s) if !strings.Contains(buf.String(), s) { From c75aed085189dd1af03e34eceee5fd9682c111ed Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 12 Mar 2018 16:28:12 +0000 Subject: [PATCH 4/5] Fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 037b5e46..d12683d2 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ In order to use the image, it is necessary to accept the terms of the IBM MQ lic * **LICENSE** - Set this to `accept` to agree to the MQ Advanced for Developers license. If you wish to see the license you can set this to `view`. * **LANG** - Set this to the language you would like the license to be printed in. * **MQ_QMGR_NAME** - Set this to the name you want your Queue Manager to be created with. -* **LOG_FORMAT** - Set this to change the format of the logs which are printed on the container's stdout. Set to "json" to use JSON format (JSON object per line); set to "basic" to use a simple human-readable. Defaults to "basic". +* **LOG_FORMAT** - Set this to change the format of the logs which are printed on the container's stdout. Set to "json" to use JSON format (JSON object per line); set to "basic" to use a simple human-readable format. Defaults to "basic". # Issues and contributions From b240a84ce009a53fee3c941ca5f696344e4c5d83 Mon Sep 17 00:00:00 2001 From: Arthur Barr Date: Mon, 12 Mar 2018 16:28:40 +0000 Subject: [PATCH 5/5] Fix occasional timing error in tests --- test/docker/docker_api_test_util.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/docker/docker_api_test_util.go b/test/docker/docker_api_test_util.go index 7c54b3ee..5409b456 100644 --- a/test/docker/docker_api_test_util.go +++ b/test/docker/docker_api_test_util.go @@ -309,13 +309,23 @@ func execContainerWithOutput(t *testing.T, cli *client.Client, ID string, user s if err != nil { t.Fatal(err) } - cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ + err = cli.ContainerExecStart(context.Background(), resp.ID, types.ExecStartCheck{ Detach: false, Tty: false, }) if err != nil { t.Fatal(err) } + // Wait for the command to finish + for { + inspect, err := cli.ContainerExecInspect(context.Background(), resp.ID) + if err != nil { + t.Fatal(err) + } + if !inspect.Running { + break + } + } buf := new(bytes.Buffer) // Each output line has a header, which needs to be removed _, err = stdcopy.StdCopy(buf, buf, hijack.Reader)