Updating to Java 11 causes Spring Cloud Vault to not find reactor.netty.http.client.HttpClient

[MaxPowerDarrel]

Working project with Java 10, tested with 2.0.5.RELEASE, 2.0.6.RELEASE and 2.1.0.M4 with Spring Cloud Finchley.SR1 and Greenwich.M1. Project works and runs as expected.

Changing only the Java version from 10 to 11 gives the error of

	at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:303)
	at org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:284)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1307)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1153)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:538)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:498)
	at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:320)
	at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:222)
	at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:318)
	at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:199)
	at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:848)
	at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:865)
	at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:548)
	at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:797)
	at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:421)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:340)
	at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:139)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.bootstrapServiceContext(BootstrapApplicationListener.java:197)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:104)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:70)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:127)
	at org.springframework.boot.context.event.EventPublishingRunListener.environmentPrepared(EventPublishingRunListener.java:75)
	at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:54)
	at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:371)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:330)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1291)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1279)
	at com.chg.documentservice.DocumentServiceApplication.main(DocumentServiceApplication.java:10)
Caused by: org.springframework.beans.BeanInstantiationException: Failed to instantiate [org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration$$EnhancerBySpringCGLIB$$99154ba1]: Constructor threw exception; nested exception is java.lang.NoClassDefFoundError: reactor/netty/http/client/HttpClient
	at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:184)
	at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:117)
	at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:299)
	... 30 common frames omitted
Caused by: java.lang.NoClassDefFoundError: reactor/netty/http/client/HttpClient
	at org.springframework.vault.config.ClientHttpConnectorFactory.create(ClientHttpConnectorFactory.java:54)
	at org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration.createConnector(VaultReactiveBootstrapConfiguration.java:105)
	at org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration.<init>(VaultReactiveBootstrapConfiguration.java:86)
	at org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration$$EnhancerBySpringCGLIB$$99154ba1.<init>(<generated>)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
	at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:172)
	... 32 common frames omitted
Caused by: java.lang.ClassNotFoundException: reactor.netty.http.client.HttpClient
	at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:583)
	at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:178)
	at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521)
	... 41 common frames omitted

This is not a Webflux project. I understand Java 11 removed certain classes from the JDK, but these are not the classes that were removed.

Edit: Styling

[wilkinsona]

I can't see the connection between the title and the failure, but this looks like a Spring Cloud Vault problem to me. VaultReactiveBootstrapConfiguration is calling org.springframework.vault.config.ClientHttpConnectorFactory which requires Reactor Netty but VaultReactiveBootstrapConfiguration isn't conditional on Reactor Netty being on the classpath. Can you please open a Spring Cloud Vault issue?

[MaxPowerDarrel]

Thank you for the reply. I apologize for the misnamed title. I was chasing a different issue and had that in my clipboard; an oversight on my part.
I opened an issue with Spring Cloud Vault.

[wilkinsona]

Thanks. For reference, here is the issue that @MaxPowerDarrel opened: spring-cloud/spring-cloud-vault#254.

[mp911de]

After an investigation, I'm not sure that Spring Cloud Vault is causing the issue. Requiring common Reactor types (Mono, HttpClient ) via @ConditionalOnClass results in a different error that is rooted in SpringBootCondition:

Stack trace excerpt:

java.lang.IllegalStateException: Error processing condition on org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration.reactiveVaultSessionManager
	at org.springframework.boot.autoconfigure.condition.SpringBootCondition.matches(SpringBootCondition.java:64)
...
Caused by: java.lang.IllegalStateException: Failed to introspect Class [org.springframework.cloud.vault.config.VaultReactiveBootstrapConfiguration] from ClassLoader [jdk.internal.loader.ClassLoaders$AppClassLoader@512ddf17]
...
Caused by: java.lang.NoClassDefFoundError: reactor/core/publisher/Mono

Full error log: https://gist.github.com/mp911de/369fb104bee3583263c226fdebb6103b

Java version:

java version "11.0.1" 2018-10-16 LTS
Java(TM) SE Runtime Environment 18.9 (build 11.0.1+13-LTS)
Java HotSpot(TM) 64-Bit Server VM 18.9 (build 11.0.1+13-LTS, mixed mode)

Auto-configuration: VaultReactiveBootstrapConfiguration.java

I tried various @ConditionalOnClass(…) combinations locally without an effect. Switching to Java 11 seems to have an impact. What is VaultReactiveBootstrapConfiguration doing wrong so that condition evaluation fails?

[wilkinsona]

It's difficult to say without seeing the code that produced the error and it sounds like you made some changes locally to produce it. Perhaps I've misunderstood? I didn't notice the significance of Java 11 when I originally triaged this issue. Spring Boot 2.0.x doesn't support Java 11. It would be interesting to know what the behaviour is with Boot 2.1.

[wilkinsona]

I think I've made some progress using a pom similar to that in the Gist shared above. The problem appears to be that the annotations on VaultReactiveBootstrapConfiguration are being introspected using reflection rather than ASM, i.e. the AnnotationMetadata is a StandardAnnotationMetadata. The use of reflection means that @ConditionalOnClass disappears as it references a class that isn't on the classpath.

[mp911de]

Sorry for confusion: The issue can be reproduced using the attached pom.xml that (generated from a plain Initializr application). I played locally with the auto-config to check whether extending @ConditionalOnClass has an effect but there was no effect on the issue.

I tried to use Spring Boot 2.1 RC1.

[wilkinsona]

StandardAnnotationMetadata is being used as VaultReactiveBootstrapConfiguration is being configured as a source on SpringApplicationBuilder. That happens due to it being listed in spring.factories under the org.springframework.cloud.bootstrap.BootstrapConfiguration key from where it's resolved as a class and then passed in as a source.

If a BootstrapConfiguration is intended to support @ConditionalOnClass then this is a general Spring Cloud problem. If it's not, then this is a Spring Cloud Vault problem.

[wilkinsona]

It works with Java 10 because VaultReactiveBootstrapConfiguration is never actually added as a source. It's skipped here:

for (String name : names) {
	Class<?> cls = ClassUtils.resolveClassName(name, null);
	try {
		cls.getDeclaredAnnotations();
	}
	catch (Exception e) {
		continue;
	}
	sources.add(cls);
}

The call to cls.getDeclaredAnnotations() throws an exception:

java.lang.ArrayStoreException: sun.reflect.annotation.TypeNotPresentExceptionProxy
	at java.base/sun.reflect.annotation.AnnotationParser.parseClassArray(AnnotationParser.java:724)
	at java.base/sun.reflect.annotation.AnnotationParser.parseArray(AnnotationParser.java:531)
	at java.base/sun.reflect.annotation.AnnotationParser.parseMemberValue(AnnotationParser.java:355)
	at java.base/sun.reflect.annotation.AnnotationParser.parseAnnotation2(AnnotationParser.java:286)
	at java.base/sun.reflect.annotation.AnnotationParser.parseAnnotations2(AnnotationParser.java:120)
	at java.base/sun.reflect.annotation.AnnotationParser.parseAnnotations(AnnotationParser.java:72)
	at java.base/java.lang.Class.createAnnotationData(Class.java:3710)
	at java.base/java.lang.Class.annotationData(Class.java:3699)
	at java.base/java.lang.Class.getDeclaredAnnotations(Class.java:3666)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.bootstrapServiceContext(BootstrapApplicationListener.java:188)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:104)
	at org.springframework.cloud.bootstrap.BootstrapApplicationListener.onApplicationEvent(BootstrapApplicationListener.java:70)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.doInvokeListener(SimpleApplicationEventMulticaster.java:172)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.invokeListener(SimpleApplicationEventMulticaster.java:165)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:139)
	at org.springframework.context.event.SimpleApplicationEventMulticaster.multicastEvent(SimpleApplicationEventMulticaster.java:127)
	at org.springframework.boot.context.event.EventPublishingRunListener.environmentPrepared(EventPublishingRunListener.java:75)
	at org.springframework.boot.SpringApplicationRunListeners.environmentPrepared(SpringApplicationRunListeners.java:54)
	at org.springframework.boot.SpringApplication.prepareEnvironment(SpringApplication.java:345)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:304)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1256)
	at org.springframework.boot.SpringApplication.run(SpringApplication.java:1244)
	at com.example.demo.Gh14873Application.main(Gh14873Application.java:10)

With Java 11 the behaviour is different and, rather than throwing an exception, the annotation is dropped. This is looking more like a Java 11-specific bug in BootstrapApplicationListener.