diff --git a/spring-integration-amqp/src/test/java/org/springframework/integration/amqp/rule/BrokerRunning.java b/spring-integration-amqp/src/test/java/org/springframework/integration/amqp/rule/BrokerRunning.java index 71e55e4ebe6..7de98161bb8 100644 --- a/spring-integration-amqp/src/test/java/org/springframework/integration/amqp/rule/BrokerRunning.java +++ b/spring-integration-amqp/src/test/java/org/springframework/integration/amqp/rule/BrokerRunning.java @@ -132,12 +132,15 @@ public Statement apply(Statement base, Description description) { return super.apply(base, description); } - public void removeTestQueues() { + public void removeTestQueues(String... additionalQueues) { CachingConnectionFactory connectionFactory = new CachingConnectionFactory("localhost"); RabbitAdmin admin = new RabbitAdmin(connectionFactory); for (Queue queue : this.queues) { admin.deleteQueue(queue.getName()); } + for (String queue: additionalQueues) { + admin.deleteQueue(queue); + } connectionFactory.destroy(); } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/support/json/BoonJsonObjectMapper.java b/spring-integration-core/src/main/java/org/springframework/integration/support/json/BoonJsonObjectMapper.java index accb5e798d6..12b514a17d0 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/support/json/BoonJsonObjectMapper.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/support/json/BoonJsonObjectMapper.java @@ -1,5 +1,5 @@ /* - * Copyright 2014-2016 the original author or authors. + * Copyright 2014-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Map; import java.util.concurrent.Executors; +import java.util.function.Consumer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,6 +38,7 @@ import org.boon.json.JsonSerializerFactory; import org.boon.json.JsonSlurper; import org.boon.json.ObjectMapper; +import org.boon.json.implementation.ObjectMapperImpl; import org.springframework.beans.factory.BeanClassLoaderAware; import org.springframework.integration.mapping.support.JsonHeaders; @@ -46,6 +48,7 @@ * The Boon (@link https://github.com/RichardHightower/boon) {@link JsonObjectMapper} implementation. * * @author Artem Bilan + * @author Gary Russell * @since 4.1 */ public class BoonJsonObjectMapper extends JsonObjectMapperAdapter, Object> @@ -67,6 +70,18 @@ public BoonJsonObjectMapper() { this.objectMapper = JsonFactory.create(); } + public BoonJsonObjectMapper(Consumer jpfConfig, Consumer jsfConfig) { + JsonParserFactory jpf = new JsonParserFactory(); + if (jpfConfig != null) { + jpfConfig.accept(jpf); + } + JsonSerializerFactory jsf = new JsonSerializerFactory(); + if (jsfConfig != null) { + jsfConfig.accept(jsf); + } + this.objectMapper = new ObjectMapperImpl(jpf, jsf); + } + public BoonJsonObjectMapper(JsonParserFactory parserFactory, JsonSerializerFactory serializerFactory) { this.objectMapper = JsonFactory.create(parserFactory, serializerFactory); } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/support/json/JacksonJsonUtils.java b/spring-integration-core/src/main/java/org/springframework/integration/support/json/JacksonJsonUtils.java index e26d0404633..0e4b542c5c9 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/support/json/JacksonJsonUtils.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/support/json/JacksonJsonUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,16 +38,8 @@ private JacksonJsonUtils() { ClassUtils.isPresent("com.fasterxml.jackson.databind.ObjectMapper", classLoader) && ClassUtils.isPresent("com.fasterxml.jackson.core.JsonGenerator", classLoader); - private static final boolean jacksonPresent = - ClassUtils.isPresent("org.codehaus.jackson.map.ObjectMapper", classLoader) && - ClassUtils.isPresent("org.codehaus.jackson.JsonGenerator", classLoader); - public static boolean isJackson2Present() { return jackson2Present; } - public static boolean isJacksonPresent() { - return jacksonPresent; - } - } diff --git a/spring-integration-core/src/main/java/org/springframework/integration/support/json/JsonObjectMapperProvider.java b/spring-integration-core/src/main/java/org/springframework/integration/support/json/JsonObjectMapperProvider.java index 7967a7c96c6..a9c38712e9e 100644 --- a/spring-integration-core/src/main/java/org/springframework/integration/support/json/JsonObjectMapperProvider.java +++ b/spring-integration-core/src/main/java/org/springframework/integration/support/json/JsonObjectMapperProvider.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,13 @@ import org.springframework.util.ClassUtils; +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * Simple factory to provide {@linkplain JsonObjectMapper} * instances dependently of jackson-databind or boon libs in the classpath. @@ -60,6 +67,25 @@ else if (boonPresent) { } } + /** + * Return an object mapper builder if available. + * @param preferBoon true to prefer boon if available. + * @return the mapper builder. + * @throws IllegalStateException if an implementation is not available. + * @since 5.0 + */ + public static JsonObjectMapperBuilder newInstanceBuilder(boolean preferBoon) { + if (JacksonJsonUtils.isJackson2Present() && (!preferBoon || !boonPresent)) { + return new JacksonJsonObjectMapperBuilder(); + } + else if (boonPresent) { + return new BoonJsonObjectMapperBuilder(); + } + else { + throw new IllegalStateException("Neither jackson-databind.jar, nor boon.jar is present in the classpath."); + } + } + /** * Returns true if a supported JSON implementation is on the class path. * @return true if {@link #newInstance()} will return a mapper. @@ -69,4 +95,76 @@ public static boolean jsonAvailable() { return JacksonJsonUtils.isJackson2Present() || boonPresent; } + public static abstract class JsonObjectMapperBuilder> { + + protected boolean usePropertyOnly = true; // NOSONAR + + protected boolean includeAllValues = true; // NOSONAR + + public B usePropertyOnly(boolean use) { + this.usePropertyOnly = use; + return _this(); + } + + public B includeAllValues(boolean include) { + this.includeAllValues = include; + return _this(); + } + + @SuppressWarnings("unchecked") + protected final B _this() { + return (B) this; + } + + public abstract JsonObjectMapper build(); + + } + + private static class JacksonJsonObjectMapperBuilder + extends JsonObjectMapperBuilder { + + JacksonJsonObjectMapperBuilder() { + super(); + } + + @Override + public JsonObjectMapper build() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(MapperFeature.DEFAULT_VIEW_INCLUSION, false); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + if (this.usePropertyOnly) { + objectMapper.setVisibility(PropertyAccessor.GETTER, Visibility.PUBLIC_ONLY); + objectMapper.setVisibility(PropertyAccessor.FIELD, Visibility.NONE); + } + if (this.includeAllValues) { + objectMapper.setSerializationInclusion(JsonInclude.Include.ALWAYS); + } + return new Jackson2JsonObjectMapper(objectMapper); + } + + } + + private static class BoonJsonObjectMapperBuilder extends JsonObjectMapperBuilder { + + BoonJsonObjectMapperBuilder() { + super(); + } + + @Override + public JsonObjectMapper build() { + return new BoonJsonObjectMapper(null, f -> { + f.useAnnotations(); + if (this.usePropertyOnly) { + f.usePropertyOnly(); + } + if (this.includeAllValues) { + f.includeDefaultValues() + .includeNulls() + .includeEmpty(); + } + }); + } + + } + } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java b/spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java index 36bee26abb6..0390abcce05 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2016 the original author or authors. + * Copyright 2002-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,4 +47,10 @@ public abstract class FileHeaders { */ public static final String MARKER = PREFIX + "marker"; + /** + * JSON representation of remote file information (if a JSON object mapper is + * available). + */ + public static final String REMOTE_FILE_INFO = PREFIX + "remoteFileInfo"; + } diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java index 2029397fc07..df0f258df05 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2016-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,6 +38,10 @@ import org.springframework.integration.file.filters.FileListFilter; import org.springframework.integration.file.filters.ReversibleFileListFilter; import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.support.AbstractIntegrationMessageBuilder; +import org.springframework.integration.support.json.JacksonJsonUtils; +import org.springframework.integration.support.json.JsonObjectMapper; +import org.springframework.integration.support.json.JsonObjectMapperProvider; import org.springframework.messaging.MessagingException; import org.springframework.util.Assert; @@ -58,6 +62,7 @@ public abstract class AbstractRemoteFileStreamingMessageSource private final Comparator> comparator; + private final JsonObjectMapper objectMapper; /** * the path on the remote server. */ @@ -74,6 +79,12 @@ protected AbstractRemoteFileStreamingMessageSource(RemoteFileTemplate templat Comparator> comparator) { this.remoteFileTemplate = template; this.comparator = comparator; + if (JacksonJsonUtils.isJackson2Present()) { + this.objectMapper = JsonObjectMapperProvider.newInstanceBuilder(false).build(); + } + else { + this.objectMapper = null; + } } /** @@ -116,6 +127,16 @@ protected RemoteFileTemplate getRemoteFileTemplate() { return this.remoteFileTemplate; } + /** + * Override this method if you wish to provide your own object mapper for + * the {@link AbstractFileInfo} header. + * @return the object mapper. + * @since 5.0 + */ + protected JsonObjectMapper getObjectMapper() { + return this.objectMapper; + } + @Override public final void afterPropertiesSet() { Assert.state(this.remoteDirectoryExpression != null, "'remoteDirectoryExpression' must not be null"); @@ -136,11 +157,20 @@ protected Object doReceive() { String remotePath = remotePath(file); Session session = this.remoteFileTemplate.getSession(); try { - return getMessageBuilderFactory().withPayload(session.readRaw(remotePath)) + AbstractIntegrationMessageBuilder builder = getMessageBuilderFactory() + .withPayload(session.readRaw(remotePath)) .setHeader(IntegrationMessageHeaderAccessor.CLOSEABLE_RESOURCE, session) .setHeader(FileHeaders.REMOTE_DIRECTORY, file.getRemoteDirectory()) - .setHeader(FileHeaders.REMOTE_FILE, file.getFilename()) - .build(); + .setHeader(FileHeaders.REMOTE_FILE, file.getFilename()); + if (getObjectMapper() != null) { + try { + builder.setHeader(FileHeaders.REMOTE_FILE_INFO, getObjectMapper().toJson(file)); + } + catch (Exception e) { + logger.info("Failed to transform file info to json: " + file, e); + } + } + return builder.build(); } catch (IOException e) { throw new MessagingException("IOException when retrieving " + remotePath, e); diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java index e90e864f876..a186db846b2 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/StreamingInboundTests.java @@ -16,8 +16,10 @@ package org.springframework.integration.file.remote; +import static org.hamcrest.CoreMatchers.containsString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.mockito.BDDMockito.given; import static org.mockito.BDDMockito.willReturn; import static org.mockito.BDDMockito.willThrow; @@ -37,6 +39,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.beans.factory.BeanFactory; import org.springframework.integration.IntegrationMessageHeaderAccessor; import org.springframework.integration.channel.QueueChannel; @@ -45,6 +48,7 @@ import org.springframework.integration.file.remote.session.Session; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.file.splitter.FileSplitter; +import org.springframework.integration.support.json.JsonObjectMapperProvider; import org.springframework.integration.transformer.StreamTransformer; import org.springframework.messaging.Message; import org.springframework.messaging.MessagingException; @@ -73,14 +77,37 @@ public void testAllData() throws Exception { assertEquals("foo\nbar", new String(received.getPayload())); assertEquals("/foo", received.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); assertEquals("foo", received.getHeaders().get(FileHeaders.REMOTE_FILE)); + String fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"/foo")); + assertThat(fileInfo, containsString("permissions\":\"-rw-rw-rw")); + assertThat(fileInfo, containsString("size\":42")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\"foo")); + assertThat(fileInfo, containsString("modified\":42000")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo")); + assertThat(fileInfo, containsString("name=/foo/foo")); // close after list, transform verify(new IntegrationMessageHeaderAccessor(received).getCloseableResource(), times(2)).close(); + new DirectFieldAccessor(streamer).setPropertyValue("objectMapper", + JsonObjectMapperProvider.newInstanceBuilder(true).build()); + received = (Message) this.transformer.transform(streamer.receive()); assertEquals("baz\nqux", new String(received.getPayload())); assertEquals("/foo", received.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); assertEquals("bar", received.getHeaders().get(FileHeaders.REMOTE_FILE)); + fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"/foo")); + assertThat(fileInfo, containsString("permissions\":\"-rw-rw-rw")); + assertThat(fileInfo, containsString("size\":42")); + assertThat(fileInfo, containsString("directory\":false")); // Boon doesn't emit default values (false) + assertThat(fileInfo, containsString("filename\":\"bar")); + assertThat(fileInfo, containsString("modified\":42000")); + assertThat(fileInfo, containsString("link\":false")); // Boon doesn't emit default values (false) + assertThat(fileInfo, containsString("fileInfo")); + assertThat(fileInfo, containsString("name=/foo/bar")); // close after transform verify(new IntegrationMessageHeaderAccessor(received).getCloseableResource(), times(3)).close(); @@ -213,12 +240,12 @@ public boolean isLink() { @Override public long getSize() { - return 0; + return 42; } @Override public long getModified() { - return 0; + return 42_000; } @Override @@ -228,12 +255,16 @@ public String getFilename() { @Override public String getPermissions() { - return null; + return "-rw-rw-rw"; } @Override public String getFileInfo() { - return null; + return asString(); + } + + private String asString() { + return "StringFileInfo [name=" + this.name + "]"; } } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java index 7954be4db78..ddac25f4c4f 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/inbound/FtpStreamingMessageSourceTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.ftp.inbound; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -27,6 +28,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -35,12 +37,14 @@ import org.springframework.integration.channel.QueueChannel; import org.springframework.integration.config.EnableIntegration; import org.springframework.integration.core.MessageSource; +import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.ftp.FtpTestSupport; import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter; import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; import org.springframework.integration.metadata.SimpleMetadataStore; import org.springframework.integration.scheduling.PollerMetadata; +import org.springframework.integration.support.json.JsonObjectMapperProvider; import org.springframework.integration.transformer.StreamTransformer; import org.springframework.messaging.Message; import org.springframework.messaging.PollableChannel; @@ -61,18 +65,69 @@ public class FtpStreamingMessageSourceTests extends FtpTestSupport { @Autowired public PollableChannel data; + @Autowired + public PollableChannel dataBoon; + @SuppressWarnings("unchecked") @Test public void testAllContents() { Message received = (Message) this.data.receive(10000); assertNotNull(received); assertThat(new String(received.getPayload()), equalTo("source1")); + String fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"ftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-------")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\" ftpSource1.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo")); received = (Message) this.data.receive(10000); assertNotNull(received); assertThat(new String(received.getPayload()), equalTo("source2")); + fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"ftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-------")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\"ftpSource2.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo")); assertNull(this.data.receive(10)); } + @SuppressWarnings("unchecked") + @Test + public void testAllContentsBoon() { + Message received = (Message) this.dataBoon.receive(10000); + assertNotNull(received); + assertThat(new String(received.getPayload()), equalTo("source1")); + String fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"ftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-------")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\" ftpSource1.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo")); + received = (Message) this.dataBoon.receive(10000); + assertNotNull(received); + assertThat(new String(received.getPayload()), equalTo("source2")); + fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"ftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-------")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\"ftpSource2.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo")); + assertNull(this.dataBoon.receive(10)); + } + @Configuration @EnableIntegration public static class Config { @@ -82,6 +137,11 @@ public QueueChannel data() { return new QueueChannel(); } + @Bean + public QueueChannel dataBoon() { + return new QueueChannel(); + } + @Bean(name = PollerMetadata.DEFAULT_POLLER) public PollerMetadata defaultPoller() { PollerMetadata pollerMetadata = new PollerMetadata(); @@ -99,12 +159,29 @@ public MessageSource ftpMessageSource() { return messageSource; } + @Bean + @InboundChannelAdapter(channel = "streamBoon") + public MessageSource ftpMessageSourceBoon() { + FtpStreamingMessageSource messageSource = new FtpStreamingMessageSource(template(), null); + new DirectFieldAccessor(messageSource).setPropertyValue("objectMapper", + JsonObjectMapperProvider.newInstanceBuilder(true).build()); + messageSource.setRemoteDirectory("ftpSource/"); + messageSource.setFilter(new FtpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "streaming")); + return messageSource; + } + @Bean @Transformer(inputChannel = "stream", outputChannel = "data") public org.springframework.integration.transformer.Transformer transformer() { return new StreamTransformer(); } + @Bean + @Transformer(inputChannel = "streamBoon", outputChannel = "dataBoon") + public org.springframework.integration.transformer.Transformer transformerBoon() { + return new StreamTransformer(); + } + @Bean public FtpRemoteFileTemplate template() { return new FtpRemoteFileTemplate(ftpSessionFactory()); diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpFileInfo.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpFileInfo.java index 39c1df434a4..45c4f466bb4 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpFileInfo.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/session/SftpFileInfo.java @@ -45,6 +45,7 @@ public SftpFileInfo(LsEntry lsEntry) { /** * @see com.jcraft.jsch.SftpATTRS#isDir() */ + @Override public boolean isDirectory() { return this.attrs.isDir(); } @@ -52,6 +53,7 @@ public boolean isDirectory() { /** * @see com.jcraft.jsch.SftpATTRS#isLink() */ + @Override public boolean isLink() { return this.attrs.isLink(); } @@ -59,6 +61,7 @@ public boolean isLink() { /** * @see com.jcraft.jsch.SftpATTRS#getSize() */ + @Override public long getSize() { return this.attrs.getSize(); } @@ -66,6 +69,7 @@ public long getSize() { /** * @see com.jcraft.jsch.SftpATTRS#getMTime() */ + @Override public long getModified() { return ((long) this.attrs.getMTime()) * 1000; } @@ -73,14 +77,17 @@ public long getModified() { /** * @see com.jcraft.jsch.ChannelSftp.LsEntry#getFilename() */ + @Override public String getFilename() { return this.lsEntry.getFilename(); } + @Override public String getPermissions() { return this.attrs.getPermissionsString(); } + @Override public LsEntry getFileInfo() { return this.lsEntry; } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java index 49dc3d92b42..75fd5a40495 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/inbound/SftpStreamingMessageSourceTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.sftp.inbound; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -34,12 +35,15 @@ import org.springframework.integration.channel.QueueChannel; import org.springframework.integration.config.EnableIntegration; import org.springframework.integration.core.MessageSource; +import org.springframework.integration.file.FileHeaders; import org.springframework.integration.file.remote.session.SessionFactory; import org.springframework.integration.metadata.SimpleMetadataStore; import org.springframework.integration.scheduling.PollerMetadata; import org.springframework.integration.sftp.SftpTestSupport; import org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter; import org.springframework.integration.sftp.session.SftpRemoteFileTemplate; +import org.springframework.integration.support.json.JsonObjectMapper; +import org.springframework.integration.support.json.JsonObjectMapperProvider; import org.springframework.integration.transformer.StreamTransformer; import org.springframework.messaging.Message; import org.springframework.messaging.PollableChannel; @@ -62,18 +66,73 @@ public class SftpStreamingMessageSourceTests extends SftpTestSupport { @Autowired public PollableChannel data; + @Autowired + public PollableChannel dataBoon; + @SuppressWarnings("unchecked") @Test public void testAllContents() { Message received = (Message) this.data.receive(10000); assertNotNull(received); assertThat(new String(received.getPayload()), equalTo("source1")); + String fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"sftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-r--r--")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\" sftpSource1.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo\":")); + assertThat(fileInfo, containsString("attrs\":")); received = (Message) this.data.receive(10000); assertNotNull(received); + fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"sftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-r--r--")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\"sftpSource2.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo\":")); + assertThat(fileInfo, containsString("attrs\":")); assertThat(new String(received.getPayload()), equalTo("source2")); assertNull(this.data.receive(10)); } + @SuppressWarnings("unchecked") + @Test + public void testAllContentsBoon() { + Message received = (Message) this.dataBoon.receive(10000); + assertNotNull(received); + assertThat(new String(received.getPayload()), equalTo("source1")); + String fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"sftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-r--r--")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\" sftpSource1.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo\":")); +// assertThat(fileInfo, containsString("attrs\":")); // Boon does not reliably include this for some reason + received = (Message) this.dataBoon.receive(10000); + assertNotNull(received); + fileInfo = (String) received.getHeaders().get(FileHeaders.REMOTE_FILE_INFO); + assertThat(fileInfo, containsString("remoteDirectory\":\"sftpSource")); + assertThat(fileInfo, containsString("permissions\":\"-rw-r--r--")); + assertThat(fileInfo, containsString("size\":7")); + assertThat(fileInfo, containsString("directory\":false")); + assertThat(fileInfo, containsString("filename\":\"sftpSource2.txt")); + assertThat(fileInfo, containsString("modified\":")); + assertThat(fileInfo, containsString("link\":false")); + assertThat(fileInfo, containsString("fileInfo\":")); +// assertThat(fileInfo, containsString("attrs\":")); // Boon does not reliably include this for some reason + assertThat(new String(received.getPayload()), equalTo("source2")); + assertNull(this.dataBoon.receive(10)); + } + @Configuration @EnableIntegration public static class Config { @@ -83,6 +142,11 @@ public QueueChannel data() { return new QueueChannel(); } + @Bean + public QueueChannel dataBoon() { + return new QueueChannel(); + } + @Bean(name = PollerMetadata.DEFAULT_POLLER) public PollerMetadata defaultPoller() { PollerMetadata pollerMetadata = new PollerMetadata(); @@ -93,19 +157,42 @@ public PollerMetadata defaultPoller() { @Bean @InboundChannelAdapter(channel = "stream") - public MessageSource ftpMessageSource() { + public MessageSource sftpMessageSource() { SftpStreamingMessageSource messageSource = new SftpStreamingMessageSource(template(), null); messageSource.setRemoteDirectory("sftpSource/"); messageSource.setFilter(new SftpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "streaming")); return messageSource; } + @Bean + @InboundChannelAdapter(channel = "streamBoon") + public MessageSource sftpMessageSourceBoon() { + final JsonObjectMapper mapper = JsonObjectMapperProvider.newInstanceBuilder(true).build(); + SftpStreamingMessageSource messageSource = new SftpStreamingMessageSource(template(), null) { + + @Override + protected JsonObjectMapper getObjectMapper() { + return mapper; + } + + }; + messageSource.setRemoteDirectory("sftpSource/"); + messageSource.setFilter(new SftpPersistentAcceptOnceFileListFilter(new SimpleMetadataStore(), "streaming")); + return messageSource; + } + @Bean @Transformer(inputChannel = "stream", outputChannel = "data") public org.springframework.integration.transformer.Transformer transformer() { return new StreamTransformer(); } + @Bean + @Transformer(inputChannel = "streamBoon", outputChannel = "dataBoon") + public org.springframework.integration.transformer.Transformer transformerBoon() { + return new StreamTransformer(); + } + @Bean public SftpRemoteFileTemplate template() { return new SftpRemoteFileTemplate(ftpSessionFactory()); diff --git a/src/reference/asciidoc/ftp.adoc b/src/reference/asciidoc/ftp.adoc index 0ebaad2ec1a..415737a2dbd 100644 --- a/src/reference/asciidoc/ftp.adoc +++ b/src/reference/asciidoc/ftp.adoc @@ -534,7 +534,13 @@ If you don't actually want to persist the state, an in-memory `SimpleMetadataSto If you wish to use a filename pattern (or regex) as well, use a `CompositeFileListFilter`. The java configuration below shows one technique to remove the remote file after processing. -Use the `max-fetch-size` attribute to limit the number of files fetched on each poll when a fetch is necessary; set to 1 and use a persistent filter when running in a clustered environment; +Use the `max-fetch-size` attribute to limit the number of files fetched on each poll when a fetch is necessary; set to 1 and use a persistent filter when running in a clustered environment. + +The adapter puts the remote directory and file name in headers `FileHeaders.REMOTE_DIRECTORY` and `FileHeaders.REMOTE_FILE` respectively. +Starting with _version 5.0_, complete remote file information, in JSON, is provided in the `FileHeaders.REMOTE_FILE_INFO` header, if a supported JSON object mapper on the classpath - currently Jackson 2 and Boon. + +Common properties are available as top level map entries, but the complete `FTPFile` object is included, under key `fileInfo`, providing all properties supported by the underlying library. +While there is some duplication with the top-level entries, the top level entries provide a consistent attribute set between FTP and SFTP. ==== Configuring with Java Configuration diff --git a/src/reference/asciidoc/sftp.adoc b/src/reference/asciidoc/sftp.adoc index 2cf2b4d6fe8..4bc94d977fa 100644 --- a/src/reference/asciidoc/sftp.adoc +++ b/src/reference/asciidoc/sftp.adoc @@ -573,7 +573,13 @@ If you don't actually want to persist the state, an in-memory `SimpleMetadataSto If you wish to use a filename pattern (or regex) as well, use a `CompositeFileListFilter`. The java configuration below shows one technique to remove the remote file after processing. -Use the `max-fetch-size` attribute to limit the number of files fetched on each poll when a fetch is necessary; set to 1 and use a persistent filter when running in a clustered environment; +Use the `max-fetch-size` attribute to limit the number of files fetched on each poll when a fetch is necessary; set to 1 and use a persistent filter when running in a clustered environment. + +The adapter puts the remote directory and file name in headers `FileHeaders.REMOTE_DIRECTORY` and `FileHeaders.REMOTE_FILE` respectively. +Starting with _version 5.0_, complete remote file information, in JSON, is provided in the `FileHeaders.REMOTE_FILE_INFO` header, if a supported JSON object mapper on the classpath - currently Jackson 2 and Boon (Jackson is recommended). + +Common properties are available as top level map entries, but the complete JSch `LsEntry` object is included, under key `fileInfo`, providing all properties supported by the underlying library. +While there is some duplication with the top-level entries, the top level entries provide a consistent attribute set between FTP and SFTP. ==== Configuring with Java Configuration diff --git a/src/reference/asciidoc/whats-new.adoc b/src/reference/asciidoc/whats-new.adoc index 8554f7f0412..86df0530eca 100644 --- a/src/reference/asciidoc/whats-new.adoc +++ b/src/reference/asciidoc/whats-new.adoc @@ -82,6 +82,9 @@ See <> and <> for more information. The FTP and SFTP outbound gateways now support the `REPLACE_IF_MODIFIED` `FileExistsMode` when fetching remote files. See <> and <> for more information. +If Jackson (or Boon) is on the classpath, the (S)FTP streaming inbound channel adapters now add a JSON representation of the remote file information in a message header. +See <> and <> for more information. + ==== Integration Properties Since _version 4.3.2_ a new `spring.integration.readOnly.headers` global property has been added to customize the list of headers which should not be copied to a newly created `Message` by the `MessageBuilder`.