From 7a5915d929db070186ed6d831d3de3d0d785df40 Mon Sep 17 00:00:00 2001 From: ibmmqmet Date: Thu, 10 Oct 2019 13:29:04 +0100 Subject: [PATCH] Add timeout exploitation --- BUILDING.md | 55 +++++++++ CHANGES.md | 69 ++++++----- README.md | 109 +++++++++--------- VERSION | 2 +- build.gradle | 32 +++-- .../boot/MQConfigurationListenerDefault.java | 101 ++++++++++++++++ .../boot/MQConfigurationProperties.java | 3 +- .../main/resources/META-INF/spring.factories | 2 + 8 files changed, 271 insertions(+), 102 deletions(-) create mode 100644 BUILDING.md create mode 100644 mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationListenerDefault.java diff --git a/BUILDING.md b/BUILDING.md new file mode 100644 index 0000000..8ee2ccb --- /dev/null +++ b/BUILDING.md @@ -0,0 +1,55 @@ +# Building the library from source + +The library can be built directly using Gradle from the build.gradle control +file. The output can be copied to a local directory or pushed to a Maven repository. + +Make sure the VERSION is correctly set for the output jar file, and that versions +of the dependencies in the build.gradle file are also those desired. + +Then run gradlew to compile: + +``` + export PushToMaven=true # if we want to push to a Maven repository + ./gradlew -i --rerun-tasks uploadArchives +``` + +## Building for a Maven repository + +The VERSION file in this directory contains the version number associated with the build. +For example, "0.1.2-SNAPSHOT". + +Output from the build can be uploaded to a Maven repository. + +The uploadArchives task controls publishing of the output. It uses the VERSION number to +determine what to do, along with an environment variable. +This means that we can build a non-SNAPSHOT version while still not pushing it out and +the github version of the file can match exactly what was built. + +- If the version contains 'SNAPSHOT' that we will use that temporary repo in the Central Repository. + else we push to the RELEASE repository +- If the version contains 'LOCAL' or the environment variable "PushToMaven" is not set + ** then the output will be copied to a local Maven repository + under the user's home directory (~/.m2/repository). + ** otherwise we attempt to push the jar files to the Nexus Central Repository. + + +## Releasing from Nexus +If pushing to the Nexus Release area, then once the build has been successfully transferred +you must log into Nexus to do the final promotion (CLOSE/RELEASE) of the artifact. Although it is +possible to automate that process, I am not doing it in this build file so we do a manual check +that the build has been successful and to check validity before freezing a version number. + +Using Nexus Central Repository requires authentication and authorisation. The userid and password +associated with the account are held in a local file (gradle.properties) that is not part +of this public repository. That properties file also holds information about the signing key that Nexus +requires. + + ---- Example gradle.properties file -------- + # These access the GPG key and certificate + signing.keyId=AAA111BB + signing.password=MyPassw0rd + signing.secretKeyRingFile=/home/user/.gnupg/secring.gpg + # This is the authentication to Nexus + ossrhUsername=myNexusId + ossrhPassword=MyOtherPassw0rd + -------------------------------------------- diff --git a/CHANGES.md b/CHANGES.md index b7f75ce..31b7bfc 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,48 +1,53 @@ -# 0.0.1 (2018-01-18) -- [NEW] Initial skeleton release. +# 2.1.3 (2019-xx-xx) +- Update dependencies to MQ 9.1.3 +- Update dependencies to spring boot 2.x.x +- Override for polling listener default timeout -# 0.0.2 (2018-01-22) -- Modify default config to match MQ Docker container image defaults -- add TLS related properties +# 2.1.2 +- Add bean instantiation conditions keeping the correct order - XAConnectionFactoryWrapper and after this IBM connection factory +- Add XA wrapper functionality for external JTA providers and assign the nonXA connection factory +- Update dependencies to spring boot 2.1.4 -# 0.0.3 (2018-03-08) -- Change configuration prefix to "ibm.mq" to reduce ambiguities if you have other messaging attributes in the same application -- Modify build.gradle to make pushing to Maven a more explicit operation +# 2.1.1 +- Update dependencies to MQ 9.1.2 +- Update dependencies to spring boot 2.1.3 +- Add applicationName configuration property (#20) -# 0.0.4 (2018-04-02) -- Allow USER_AUTHENTICATION_MQCSP to be configured from properties +# 2.1.0 +- Simplify connection pool creation using spring boot 2.1.0 resources +- Update dependencies to MQ 9.1.1 -# 2.0.0 (2018-05-27) -- Upgrade the spring boot dependency to spring boot 2.0.2 -- Upgrade plugin version to 2.0.0 (according to spring boot version 2.x) +# 2.0.9 +- Update dependencies to spring boot 2.1.0 -# 2.0.1 (2018-09-14) -- Added ability to set a client id on the connection +# 2.0.8 +- Add pooled connection factory option + +# 2.0.7 (2018-10-19) +- Replace a broken 2.0.6 # 2.0.5 (2018-10-03) - Update dependencies to spring boot 2.0.5 - Add CCDTUrl and SSLPeer to configurable properties - Make MQConnectionFactoryFactory a public class (see issue #7) -# 2.0.7 (2018-10-19) -- Replace a broken 2.0.6 +# 2.0.1 (2018-09-14) +- Added ability to set a client id on the connection -# 2.0.8 -- Add pooled connection factory option +# 2.0.0 (2018-05-27) +- Upgrade the spring boot dependency to spring boot 2.0.2 +- Upgrade plugin version to 2.0.0 (according to spring boot version 2.x) -# 2.0.9 -- Update dependencies to spring boot 2.1.0 +# 0.0.4 (2018-04-02) +- Allow USER_AUTHENTICATION_MQCSP to be configured from properties -# 2.1.0 -- Simplify connection pool creation using spring boot 2.1.0 resources -- Update dependencies to MQ 9.1.1 +# 0.0.3 (2018-03-08) +- Change configuration prefix to "ibm.mq" to reduce ambiguities if you have other messaging attributes in the same application +- Modify build.gradle to make pushing to Maven a more explicit operation -# 2.1.1 -- Update dependencies to MQ 9.1.2 -- Update dependencies to spring boot 2.1.3 -- Add applicationName configuration property (#20) +# 0.0.2 (2018-01-22) +- Modify default config to match MQ Docker container image defaults +- add TLS related properties -# 2.1.2 -- Add bean instantiation conditions keeping the correct order - XAConnectionFactoryWrapper and after this IBM connection factory -- Add XA wrapper functionality for external JTA providers and assign the nonXA connection factory -- Update dependencies to spring boot 2.1.4 +# 0.0.1 (2018-01-18) +- [NEW] Initial skeleton release. diff --git a/README.md b/README.md index f9fe0e7..e0c03c8 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,8 @@ The library contains: ## Installation and Usage -If the VERSION file contains "LOCAL" in the version definition, then the gradle build process puts the -generated jar into your local directory tree. It can then be referenced from the application build. - -Otherwise it can be automatically downloaded from Maven Central. +The compiled version of this package can be automatically downloaded from Maven Central. For local modifications +and building it yourself, see [BUILDING.md](./BUILDING.md) ### Spring Boot Applications @@ -23,7 +21,7 @@ Gradle: } dependencies { - compile group: 'com.ibm.mq', name: 'mq-jms-spring-boot-starter', version: '2.1.1' + compile group: 'com.ibm.mq', name: 'mq-jms-spring-boot-starter', version: '2.1.3' } Maven: @@ -32,13 +30,13 @@ Maven: com.ibm.mq mq-jms-spring-boot-starter - 2.1.1 + 2.1.3 ``` -**Note** This repository and the corresponding Maven Central artifact has now been upgraded for -Spring Boot 2 applications. For Spring Boot 1, you should continue to use the previously-released -artifact at version 0.0.4. +**Note** This repository and the corresponding Maven Central artifact require +Spring Boot 2. For Spring Boot 1 compatibility, you can use the previously-released +level of the artifact at version 0.0.4. ## Design Approach @@ -47,8 +45,8 @@ The approach taken here is to follow the model for JMS applications shown in the The same application code from that example ought to work with MQ, with the simple replacement of the messaging provider in its dependency to point at this package, and changing the queue name ("mailbox" in that example) to "DEV.QUEUE.1", which is created automatically in the Docker-packaged MQ server. -Essentially what gets configured from this package is a ConnectionFactory which Spring's JmsTemplate implementation -exploits to provide a simpler interface. +Essentially what gets configured from this package are a ConnectionFactory which Spring's JmsTemplate implementation +exploits to provide a simpler interface, and a MessageListener. ## Getting Started @@ -77,7 +75,24 @@ The default attributes are ibm.mq.user=admin ibm.mq.password=passw0rd -### Extended Configuration Options +### Connection security + +The default userid and password have been chosen for a commonly-used queue manager +configuration. + +To disable user/password checking entirely, you must set the `ibm.mq.user` attribute to an empty value +so that the default is not used. + +``` + ibm.mq.user= +``` + +Of course, that level of access must be permitted by your queue manager. The usual CHLAUTH and CONNAUTH +rules will apply to assign an identity to the connection. + +Configuration of secure connections with TLS are discussed below. + +### Configuration Options If you already have a running MQ queue manager that you want to use, then you can easily modify the default configuration to match by providing override values. @@ -108,8 +123,7 @@ You will probably also need to set - `ibm.mq.user` - `ibm.mq.password` -to override the default values. These attributes can be set to an empty value, to use the local OS userid -automatically with no authentication (if the queue manager has been set up to allow that). +to override the default values. For example in an `application.properties` file: @@ -121,6 +135,18 @@ For example in an `application.properties` file: Spring Boot will then create a ConnectionFactory that can then be used to interact with your queue manager. +| Option | Description | +| --------------------------- | ------------------------------------------------------------------------------- | +| ibm.mq.queueManager | Name of queue manager | +| ibm.mq.channel | Channel Name for SVRCONN | +| ibm.mq.connName | Connection Name, which can be comma-separated list | +| ibm.mq.ccdtUrl | Location of the MQ CCDT file (URL can reference http/ftp location) | +| ibm.mq.user | User Name. Must be set to an empty value to turn off authentication attempts | +| ibm.mq.password | Password | +| ibm.mq.clientId | ClientId uniquely identifies the app connection for durable subscriptions | +| ibm.mq.applicationName | Application Name used for Uniform Cluster balancing | + + #### TLS related options The following options all default to null, but may be used to assist with configuring TLS @@ -158,52 +184,23 @@ Alternatively you may use the default Spring Caching connection factory with the | spring.jms.cache.producers | Whether to cache message producers | | spring.jms.cache.session-cache-size | Size of the session cache (per JMS Session type) | -## Related documentation - -- [MQ documentation](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.0.0/com.ibm.mq.helphome.v90.doc/WelcomePagev9r0.htm) -- [Spring Boot documentation](https://projects.spring.io/spring-boot/) -- [Spring Framework documentation](https://projects.spring.io/spring-framework/) - -# Development +### JMS Polling Listener Timer configuration -### Building for a Maven repository +The Spring AbstractPollingMessageListenerContainer interface has a default polling timer of 1 second. This can now be configured +with the `spring.jms.listener.receiveTimeout` property. If the property is not explicitly set, then this MQ Spring Boot +component resets the initial timeout value to 30 seconds which has been shown to be more cost-effective. Application code +can still set its own preferred value. -The VERSION file in this directory contains the version number associated with the build. -For example, "0.1.2-SNAPSHOT". +| Option | Description | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| spring.jms.listener.receiveTimeout | How frequently to poll for received messages. Default is 1s. Given as a Duration string: "1m", "60s", "60000" are equivalent | -Output from the build can be uploaded to a Maven repository. - -The uploadArchives task controls publishing of the output. It uses the VERSION number to -determine what to do, along with an environment variable. -This means that we can build a non-SNAPSHOT version while still not pushing it out and -the github version of the file can match exactly what was built. - -- If the version contains 'SNAPSHOT' that we will use that temporary repo in the Central Repository. - else we push to the RELEASE repository -- If the version contains 'LOCAL' or the environment variable "PushToMaven" is not set - ** then the output will be copied to a local Maven repository - under the user's home directory (~/.m2/repository). - ** otherwise we attempt to push the jar files to the Nexus Central Repository. - -If pushing to the Nexus Release area, then once the build has been successfully transferred -you must log into Nexus to do the final promotion (CLOSE/RELEASE) of the artifact. Although it is -possible to automate that process, I am not doing it in this build file so we do a manual check -that the build has been successful and to check validity before freezing a version number. +## Related documentation -Using Nexus Central Repository requires authentication and authorisation. The userid and password -associated with the account are held in a local file (gradle.properties) that is not part -of this public repository. That properties file also holds information about the signing key that Nexus -requires. +- [MQ documentation](https://www.ibm.com/support/knowledgecenter/en/SSFKSJ_9.1.0/com.ibm.mq.helphome.v91.doc/WelcomePagev9r1.htm) +- [Spring Boot documentation](https://projects.spring.io/spring-boot/) +- [Spring Framework documentation](https://projects.spring.io/spring-framework/) - ---- Example gradle.properties file -------- - # These access the GPG key and certificate - signing.keyId=AAA111BB - signing.password=MyPassw0rd - signing.secretKeyRingFile=/home/user/.gnupg/secring.gpg - # This is the authentication to Nexus - ossrhUsername=myNexusId - ossrhPassword=MyOtherPassw0rd - -------------------------------------------- ### Pull requests @@ -218,7 +215,7 @@ The preferred approach for using this package in other projects will be to use t ### License -Copyright © 2018 IBM Corp. All rights reserved. +Copyright © 2018, 2019 IBM Corp. All rights reserved. Licensed under the apache license, version 2.0 (the "license"); you may not use this file except in compliance with the license. you may obtain a copy of the license at diff --git a/VERSION b/VERSION index eca07e4..ac2cdeb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.2 +2.1.3 diff --git a/build.gradle b/build.gradle index b22c451..e81e09d 100644 --- a/build.gradle +++ b/build.gradle @@ -23,10 +23,18 @@ subprojects { apply plugin: 'maven' apply plugin: 'maven-publish' apply plugin: 'signing' - - - - group = 'com.ibm.mq' // Use this to build the package + + // Direct Dependencies - give versions here + ext.mqVersion = '9.1.3.0' + ext.springVersion = '5.2.0.BUILD-SNAPSHOT' + ext.springBootVersion = '2.2.0.BUILD-SNAPSHOT' + + ext.pooledJmsVersion = '1.0.6' + ext.jUnitVersion = '4.12' + + // The groupid for the compiled jars when uploaded + group = 'com.ibm.mq' + // The designated version is in an external file. Read its contents here. version = new File(rootDir, 'VERSION').text.trim() // If the version says "snapshot" anywhere assume it is not a release. If property or environment var is set, @@ -41,15 +49,16 @@ subprojects { repositories { mavenLocal() + maven { url "https://repo.spring.io/snapshot" } + maven { url "https://repo.spring.io/milestone" } + mavenCentral() + } - // Common dependencies - ext.springVersion = '5.1.6.RELEASE' - ext.springBootVersion = '2.1.4.RELEASE' dependencies { // MQ Java client accessed from Maven Central Repository - compile group: 'com.ibm.mq', name: 'com.ibm.mq.allclient', version: '9.1.2.0' + compile group: 'com.ibm.mq', name: 'com.ibm.mq.allclient', version: mqVersion // Spring compile group: 'org.springframework', name: 'spring-core', version: springVersion @@ -58,11 +67,11 @@ subprojects { compile group: 'org.springframework', name: 'spring-jms', version: springVersion compile group: 'org.springframework.boot', name: 'spring-boot-starter', version: springBootVersion - //pool - compile group: 'org.messaginghub', name: 'pooled-jms', version: '1.0.3' + //Pooled connections + compile group: 'org.messaginghub', name: 'pooled-jms', version: pooledJmsVersion // Testing - testCompile group: 'junit', name: 'junit', version: '4.12' + testCompile group: 'junit', name: 'junit', version: jUnitVersion testCompile group: 'org.springframework.boot', name: 'spring-boot-starter-test', version: springBootVersion } @@ -192,5 +201,4 @@ subprojects { } } } - } diff --git a/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationListenerDefault.java b/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationListenerDefault.java new file mode 100644 index 0000000..01aab29 --- /dev/null +++ b/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationListenerDefault.java @@ -0,0 +1,101 @@ +/* + * Copyright © 2018, 2019 IBM Corp. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the + * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package com.ibm.mq.spring.boot; + +import java.util.Properties; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.jms.JmsProperties; +import org.springframework.boot.context.event.ApplicationEnvironmentPreparedEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.PropertiesPropertySource; +import org.springframework.jms.annotation.JmsListener; + +/** + * The default Spring configuration for a JMS Listener sets the receive timeout (a polling + * loop timer) too low for a cost-effective solution with IBM MQ. + * See this article + * for more information. + * + * This class will override that default value if it is not explicitly set in the application + * properties. The application still has control though. If the app sets the value + * via a call to the listener.setReceiveTimeout method, then that still will be honoured as it comes + * after the creation and initial attribute setting of the listener object. + * + * This method is invoked early in the cycle, once the application environment has been loaded. We then + * get the opportunity to inspect and add to the environment that will be used later on by the + * JMS Listener initialisers. + * + * This allows the MQ JMS listener to have a different effective default than other JMS providers. + * + * Support for this property within Spring Boot itself was added in version 2.2.0 + * + */ +@ConditionalOnClass({ JmsProperties.Listener.class, JmsListener.class }) +@ConditionalOnMissingBean(JmsListener.class) +public class MQConfigurationListenerDefault implements ApplicationListener { + @Autowired + private final Long defaultReceiveTimeout = 30 * 1000L; // 30 seconds + + // There are a number of formats for the property name supported by Spring. + // This set includes the recommended variants. + private static String lcPrefix = "spring.jms.listener."; + private static String ucPrefix = "SPRING_JMS_LISTENER_"; + //@formatter:off + private final String timeoutProperties[] = { + lcPrefix + "receiveTimeout", + lcPrefix + "receivetimeout", + lcPrefix + "receive-timeout", + lcPrefix + "receive_timeout", + lcPrefix + "RECEIVE_TIMEOUT", + ucPrefix + "RECEIVE_TIMEOUT" + }; + //@formatter:on + + + @Bean + @Lazy(false) + public void onApplicationEvent(ApplicationEnvironmentPreparedEvent event) { + try { + String foundProperty = null; + Object p = null; + ConfigurableEnvironment env = (event != null) ? event.getEnvironment() : null; + if (env != null) { + // See if any of the variations of the property name exist in the environment + for (String timeoutProperty : timeoutProperties) { + p = env.getProperty(timeoutProperty); + if (p != null) { + foundProperty = timeoutProperty; + break; + } + } + + // If the user has not given any specific value for this attribute, force the new default. + if (foundProperty == null) { + Properties props = new Properties(); + props.put(timeoutProperties[0], defaultReceiveTimeout); + env.getPropertySources().addFirst(new PropertiesPropertySource(this.getClass().getName(), props)); + } + } + } + catch (Throwable e) { + // If there are any errors (there shouldn't be, but just for safety here), then ignore them. + } + } +} diff --git a/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationProperties.java b/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationProperties.java index 109dd5d..c0ae8a6 100644 --- a/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationProperties.java +++ b/mq-jms-spring-boot-starter/src/main/java/com/ibm/mq/spring/boot/MQConfigurationProperties.java @@ -52,7 +52,8 @@ public class MQConfigurationProperties { private String channel = "DEV.ADMIN.SVRCONN"; /** - * Connection Name - hostname or address and port. Format like 'system.example.com(1414)' + * Connection Name - hostname or address and port. Can be comma-separated list. + * Format like 'system.example.com(1414),system2.example.com(1414)' **/ private String connName = "localhost(1414)"; diff --git a/mq-jms-spring-boot-starter/src/main/resources/META-INF/spring.factories b/mq-jms-spring-boot-starter/src/main/resources/META-INF/spring.factories index 9028728..21dd76c 100644 --- a/mq-jms-spring-boot-starter/src/main/resources/META-INF/spring.factories +++ b/mq-jms-spring-boot-starter/src/main/resources/META-INF/spring.factories @@ -1 +1,3 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=com.ibm.mq.spring.boot.MQAutoConfiguration +org.springframework.context.ApplicationListener=com.ibm.mq.spring.boot.MQConfigurationListenerDefault +