diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index a5c182c4ab5..d37f34de145 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -66,22 +66,27 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply * Enumeration of commands supported by the gateways. */ public static enum Command { + /** * List remote files. */ LS("ls"), + /** * Retrieve a remote file. */ GET("get"), + /** * Remove a remote file (path - including wildcards). */ RM("rm"), + /** * Retrieve multiple files matching a wildcard path. */ MGET("mget"), + /** * Move (rename) a remote file. */ @@ -112,34 +117,46 @@ public static Command toCommand(String cmd) { * */ public static enum Option { + /** * Don't return full file information; just the name (ls). */ NAME_ONLY("-1"), + /** - * Include directories {@code .} and {@code ..} in the results (ls). + * Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls). */ ALL("-a"), + /** * Do not sort the results (ls with NAME_ONLY). */ NOSORT("-f"), + /** * Include directories in the results (ls). */ SUBDIRS("-dirs"), + /** * Include links in the results (ls). */ LINKS("-links"), + /** * Preserve the server timestamp (get, mget). */ PRESERVE_TIMESTAMP("-P"), + /** * Throw an exception if no files returned (mget). */ - EXCEPTION_WHEN_EMPTY("-x"); + EXCEPTION_WHEN_EMPTY("-x"), + + /** + * Recursive (ls, mget) + */ + RECURSIVE("-R"); private String option; @@ -272,9 +289,9 @@ public void setLocalFilenameGeneratorExpression(Expression localFilenameGenerato protected void onInit() { super.onInit(); Assert.notNull(this.command, "command must not be null"); - if (Command.RM.equals(this.command) || Command.MGET.equals(this.command) || + if (Command.RM.equals(this.command) || Command.GET.equals(this.command)) { - Assert.isNull(this.filter, "Filters are not supported with the rm, get, and mget commands"); + Assert.isNull(this.filter, "Filters are not supported with the rm and get commands"); } if (Command.GET.equals(this.command) || Command.MGET.equals(this.command)) { @@ -305,6 +322,11 @@ protected void onInit() { } } } + if (Command.MGET.equals(this.command)) { + Assert.isTrue(!(this.options.contains(Option.SUBDIRS)), + "Cannot use " + Option.SUBDIRS.toString() + " when using 'mget' use " + Option.RECURSIVE.toString() + + " to obtain files in subdirectories"); + } if (this.getBeanFactory() != null) { this.fileNameProcessor.setBeanFactory(this.getBeanFactory()); this.renameProcessor.setBeanFactory(this.getBeanFactory()); @@ -398,21 +420,7 @@ private Object doMv(Message requestMessage, Session session) throws IOExce } protected List ls(Session session, String dir) throws IOException { - List lsFiles = new ArrayList(); - F[] files = session.list(dir); - if (!ObjectUtils.isEmpty(files)) { - Collection filteredFiles = this.filterFiles(files); - for (F file : filteredFiles) { - if (file != null) { - if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) { - lsFiles.add(file); - } - } - } - } - else { - return lsFiles; - } + List lsFiles = listFilesInRemoteDir(session, dir, ""); if (!this.options.contains(Option.LINKS)) { purgeLinks(lsFiles); } @@ -441,6 +449,32 @@ protected List ls(Session session, String dir) throws IOException { } } + private List listFilesInRemoteDir(Session session, String directory, String subDirectory) throws IOException { + List lsFiles = new ArrayList(); + F[] files = session.list(directory + subDirectory); + boolean recursing = this.options.contains(Option.RECURSIVE); + if (!ObjectUtils.isEmpty(files)) { + Collection filteredFiles = this.filterFiles(files); + for (F file : filteredFiles) { + String fileName = this.getFilename(file); + if (file != null) { + if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) { + if (recursing && StringUtils.hasText(subDirectory)) { + lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory)); + } + else { + lsFiles.add(file); + } + } + if (recursing && this.isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) { + lsFiles.addAll(listFilesInRemoteDir(session, directory, subDirectory + fileName + File.separator)); + } + } + } + } + return lsFiles; + } + protected final List filterFiles(F[] files) { return (this.filter != null) ? this.filter.filterFiles(files) : Arrays.asList(files); } @@ -523,6 +557,22 @@ protected File get(Message message, Session session, String remoteDir, Str protected List mGet(Message message, Session session, String remoteDirectory, String remoteFilename) throws IOException { + if (this.options.contains(Option.RECURSIVE)) { + if (logger.isWarnEnabled() && !("*".equals(remoteFilename))) { + logger.warn("File name pattern must be '*' when using recursion"); + } + if (this.options.contains(Option.NAME_ONLY)) { + this.options.remove(Option.NAME_ONLY); + } + return mGetWithRecursion(message, session, remoteDirectory, remoteFilename); + } + else { + return mGetWithoutRecursion(message, session, remoteDirectory, remoteFilename); + } + } + + private List mGetWithoutRecursion(Message message, Session session, String remoteDirectory, + String remoteFilename) throws IOException { String path = this.generateFullPath(remoteDirectory, remoteFilename); String[] fileNames = session.listNames(path); if (fileNames == null) { @@ -549,6 +599,30 @@ protected List mGet(Message message, Session session, String remoteD return files; } + private List mGetWithRecursion(Message message, Session session, String remoteDirectory, + String remoteFilename) throws IOException { + List files = new ArrayList(); + @SuppressWarnings("unchecked") + List> fileNames = (List>) this.ls(session, remoteDirectory); + if (fileNames.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) { + throw new MessagingException("No files found at " + remoteDirectory + + " with pattern " + remoteFilename); + } + for (AbstractFileInfo lsEntry : fileNames) { + String fullFileName = remoteDirectory + this.getFilename(lsEntry); + /* + * With recursion, the filename might contain subdirectory information + * normalize each file separately. + */ + String fileName = this.getRemoteFilename(fullFileName); + String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName); + File file = this.get(message, session, actualRemoteDirectory, + fullFileName, fileName, false); + files.add(file); + } + return files; + } + private String getRemoteDirectory(String remoteFilePath, String remoteFilename) { String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf(remoteFilename)); if (remoteDir.length() == 0) { @@ -626,8 +700,11 @@ private String generateLocalFileName(Message message, String remoteFileName){ abstract protected String getFilename(F file); + abstract protected String getFilename(AbstractFileInfo file); + abstract protected long getModified(F file); abstract protected List> asFileInfoList(Collection files); + abstract protected F enhanceNameWithSubDirectory(F file, String directory); } diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java index 80213429b71..eca3f97fa3e 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java @@ -86,21 +86,6 @@ public void testBadFilterGet() throws Exception { } } - @Test - public void testBadFilterMGet() throws Exception { - SessionFactory sessionFactory = mock(SessionFactory.class); - TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway - (sessionFactory, "mget", "payload"); - gw.setFilter(new TestPatternFilter("")); - try { - gw.onInit(); - fail("Exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().startsWith("Filters are not supported")); - } - } - @Test public void testBadFilterRm() throws Exception { SessionFactory sessionFactory = mock(SessionFactory.class); @@ -389,9 +374,6 @@ public Object answer(InvocationOnMock invocation) throws Throwable { assertEquals("foo/bar", madeDirs.get(1)); } - /** - * @return - */ public TestLsEntry[] fileList() { TestLsEntry[] files = new TestLsEntry[6]; files[0] = new TestLsEntry("f2", 123, false, false, 1234, "-r--r--r--"); @@ -424,6 +406,83 @@ public void testLs_f() throws Exception { out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); } + public TestLsEntry[] level1List() { + return new TestLsEntry[] { + new TestLsEntry("f1", 123, false, false, 1234, "-r--r--r--"), + new TestLsEntry("d1", 0, true, false, 12345, "drw-r--r--"), + new TestLsEntry("f2", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + public TestLsEntry[] level2List() { + return new TestLsEntry[] { + new TestLsEntry("d2", 0, true, false, 12345, "drw-r--r--"), + new TestLsEntry("f3", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + public TestLsEntry[] level3List() { + return new TestLsEntry[] { + new TestLsEntry("f4", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + @Test + public void testLs_f_R() throws Exception { + SessionFactory sessionFactory = mock(SessionFactory.class); + Session session = mock(Session.class); + TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway + (sessionFactory, "ls", "payload"); + gw.setOptions("-f -R"); + gw.afterPropertiesSet(); + when(sessionFactory.getSession()).thenReturn(session); + TestLsEntry[] level1 = level1List(); + TestLsEntry[] level2 = level2List(); + TestLsEntry[] level3 = level3List(); + when(session.list("testremote/x/")).thenReturn(level1); + when(session.list("testremote/x/d1/")).thenReturn(level2); + when(session.list("testremote/x/d1/d2/")).thenReturn(level3); + @SuppressWarnings("unchecked") + Message> out = (Message>) gw + .handleRequestMessage(new GenericMessage("testremote/x")); + assertEquals(4, out.getPayload().size()); + assertEquals("f1", out.getPayload().get(0).getFilename()); + assertEquals("d1/d2/f4", out.getPayload().get(1).getFilename()); + assertEquals("d1/f3", out.getPayload().get(2).getFilename()); + assertEquals("f2", out.getPayload().get(3).getFilename()); + assertEquals("testremote/x/", + out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); + } + + @Test + public void testLs_f_R_dirs() throws Exception { + SessionFactory sessionFactory = mock(SessionFactory.class); + Session session = mock(Session.class); + TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway + (sessionFactory, "ls", "payload"); + gw.setOptions("-f -R -dirs"); + gw.afterPropertiesSet(); + when(sessionFactory.getSession()).thenReturn(session); + TestLsEntry[] level1 = level1List(); + TestLsEntry[] level2 = level2List(); + TestLsEntry[] level3 = level3List(); + when(session.list("testremote/x/")).thenReturn(level1); + when(session.list("testremote/x/d1/")).thenReturn(level2); + when(session.list("testremote/x/d1/d2/")).thenReturn(level3); + @SuppressWarnings("unchecked") + Message> out = (Message>) gw + .handleRequestMessage(new GenericMessage("testremote/x")); + assertEquals(6, out.getPayload().size()); + assertEquals("f1", out.getPayload().get(0).getFilename()); + assertEquals("d1", out.getPayload().get(1).getFilename()); + assertEquals("d1/d2", out.getPayload().get(2).getFilename()); + assertEquals("d1/d2/f4", out.getPayload().get(3).getFilename()); + assertEquals("d1/f3", out.getPayload().get(4).getFilename()); + assertEquals("f2", out.getPayload().get(5).getFilename()); + assertEquals("testremote/x/", + out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); + } + @Test public void testLs_None() throws Exception { SessionFactory sessionFactory = mock(SessionFactory.class); @@ -775,6 +834,11 @@ protected String getFilename(TestLsEntry file) { return file.getFilename(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected long getModified(TestLsEntry file) { return file.getModified(); @@ -786,18 +850,24 @@ protected List> asFileInfoList( return new ArrayList>(files); } + @Override + protected TestLsEntry enhanceNameWithSubDirectory(TestLsEntry file, String directory) { + file.setFilename(directory + file.getFilename()); + return file; + } + } class TestLsEntry extends AbstractFileInfo { - private final String filename; - private final int size; + private volatile String filename; + private final long size; private final boolean dir; private final boolean link; private final long modified; private final String permissions; - public TestLsEntry(String filename, int size, boolean dir, boolean link, + public TestLsEntry(String filename, long size, boolean dir, boolean link, long modified, String permissions) { this.filename = filename; this.size = size; @@ -835,6 +905,10 @@ public TestLsEntry getFileInfo() { return this; } + public void setFilename(String filename) { + this.filename = filename; + } + } class TestPatternFilter extends AbstractSimplePatternFileListFilter{ diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 75a1f8f8708..b55c612f5e4 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,6 +21,7 @@ import java.util.List; import org.apache.commons.net.ftp.FTPFile; + import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; @@ -28,7 +29,7 @@ /** * Outbound Gateway for performing remote file operations via FTP/FTPS. - * + * * @author Gary Russell * @since 2.1 */ @@ -54,6 +55,11 @@ protected String getFilename(FTPFile file) { return file.getName(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected long getModified(FTPFile file) { return file.getTimestamp().getTimeInMillis(); @@ -69,4 +75,11 @@ protected List> asFileInfoList(Collection fil } + @Override + protected FTPFile enhanceNameWithSubDirectory(FTPFile file, String directory) { + file.setName(directory + file.getName()); + return file; + } + + } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java similarity index 84% rename from spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java rename to spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java index 260d34d3c27..9d059f3f103 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java @@ -20,6 +20,9 @@ import java.io.IOException; import java.util.Arrays; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + import org.apache.ftpserver.FtpServer; import org.apache.ftpserver.FtpServerFactory; import org.apache.ftpserver.ftplet.Authentication; @@ -32,18 +35,25 @@ import org.apache.ftpserver.usermanager.impl.ConcurrentLoginPermission; import org.apache.ftpserver.usermanager.impl.TransferRatePermission; import org.apache.ftpserver.usermanager.impl.WritePermission; -import org.junit.rules.ExternalResource; import org.junit.rules.TemporaryFolder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.ftp.session.DefaultFtpSessionFactory; import org.springframework.integration.test.util.SocketUtils; /** + * Embedded FTP Server for test cases; exposes an associated session factory + * as a @Bean. + * * @author Artem Bilan + * @author Gary Russell * @since 3.0 */ -public class FtpServerRule extends ExternalResource { +@Configuration +public class TesFtpServer { - public static int FTP_PORT = SocketUtils.findAvailableServerSocket(); + private final int ftpPort = SocketUtils.findAvailableServerSocket(); private final TemporaryFolder ftpFolder; @@ -61,7 +71,7 @@ public class FtpServerRule extends ExternalResource { private volatile FtpServer server; - public FtpServerRule(final String root) { + public TesFtpServer(final String root) { this.ftpFolder = new TemporaryFolder() { @Override @@ -124,8 +134,22 @@ public File getTargetLocalDirectory() { return targetLocalDirectory; } - @Override - protected void before() throws Throwable { + public String getTargetLocalDirectoryName() { + return targetLocalDirectory.getAbsolutePath() + File.separator; + } + + @Bean + public DefaultFtpSessionFactory ftpSessionFactory() { + DefaultFtpSessionFactory factory = new DefaultFtpSessionFactory(); + factory.setHost("localhost"); + factory.setPort(this.ftpPort); + factory.setUsername("foo"); + factory.setPassword("foo"); + return factory; + } + + @PostConstruct + public void before() throws Throwable { this.ftpFolder.create(); this.localFolder.create(); @@ -133,7 +157,7 @@ protected void before() throws Throwable { serverFactory.setUserManager(new TestUserManager(this.ftpRootFolder.getAbsolutePath())); ListenerFactory factory = new ListenerFactory(); - factory.setPort(FTP_PORT); + factory.setPort(ftpPort); serverFactory.addListener("default", factory.createListener()); server = serverFactory.createServer(); @@ -141,8 +165,8 @@ protected void before() throws Throwable { } - @Override - protected void after() { + @PreDestroy + public void after() { this.server.stop(); this.ftpFolder.delete(); this.localFolder.delete(); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index 91feb884b4f..9fe16dd9acc 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -8,15 +8,10 @@ http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> - - - - - + + - - @@ -27,7 +22,7 @@ request-channel="inboundGet" command="get" expression="payload" - local-directory-expression="#localDir() + #remoteDirectory.toUpperCase()" + local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory.toUpperCase()" local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> @@ -46,9 +41,31 @@ request-channel="inboundMGet" command="mget" expression="payload" - local-directory-expression="#localDir() + #remoteDirectory" + local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory" + local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" + reply-channel="output"/> + + + + + + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 61fec74f09e..0519eac0d52 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.ftp.outbound; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -25,7 +26,6 @@ import org.hamcrest.Matchers; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,7 +33,7 @@ import org.springframework.integration.Message; import org.springframework.integration.channel.DirectChannel; import org.springframework.integration.core.PollableChannel; -import org.springframework.integration.ftp.FtpServerRule; +import org.springframework.integration.ftp.TesFtpServer; import org.springframework.integration.message.GenericMessage; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -46,8 +46,8 @@ @RunWith(SpringJUnit4ClassRunner.class) public class FtpServerOutboundTests { - @ClassRule - public static final FtpServerRule FTP_SERVER = new FtpServerRule(FtpServerOutboundTests.class.getSimpleName()); + @Autowired + public TesFtpServer ftpServer; @Autowired private PollableChannel output; @@ -61,10 +61,16 @@ public class FtpServerOutboundTests { @Autowired private DirectChannel inboundMGet; + @Autowired + private DirectChannel inboundMGetRecursive; + + @Autowired + private DirectChannel inboundMGetRecursiveFiltered; + @Before public void setup() { - FtpServerRule.recursiveDelete(FTP_SERVER.getTargetLocalDirectory()); - FtpServerRule.recursiveDelete(FTP_SERVER.getTargetFtpDirectory()); + TesFtpServer.recursiveDelete(ftpServer.getTargetLocalDirectory()); + TesFtpServer.recursiveDelete(ftpServer.getTargetFtpDirectory()); } @Test @@ -125,9 +131,43 @@ public void testInt2866LocalDirectoryExpressionMGET() { } } - public static String localDirectory() { - return FTP_SERVER.getTargetLocalDirectory().getAbsolutePath() + File.separator; + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursive() { + String dir = "ftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + assertEquals(3, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subFtpSource")); + } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() { + String dir = "ftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + // should have filtered ftpSource2.txt + assertEquals(2, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subFtpSource")); + + } } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java index cdb3755be94..72c7db47504 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.List; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; @@ -29,7 +30,7 @@ /** * Outbound Gateway for performing remote file operations via SFTP. - * + * * @author Gary Russell * @since 2.1 */ @@ -59,6 +60,11 @@ protected String getFilename(LsEntry file) { return file.getFilename(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected List> asFileInfoList(Collection files) { List> canonicalFiles = new ArrayList>(); @@ -73,4 +79,11 @@ protected long getModified(LsEntry file) { return ((long)file.getAttrs().getMTime()) * 1000; } + @Override + protected LsEntry enhanceNameWithSubDirectory(LsEntry file, String directory) { + DirectFieldAccessor accessor = new DirectFieldAccessor(file); + accessor.setPropertyValue("filename", directory + file.getFilename()); + return file; + } + } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml index 5c259ebbcf9..5a530a9517d 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml @@ -41,6 +41,29 @@ local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> + + + + + + + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java index 0a964c5de08..6860bdb8875 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.sftp.outbound; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -32,6 +33,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.Message; import org.springframework.integration.channel.DirectChannel; @@ -80,6 +82,9 @@ public class SftpServerOutboundTests { @Autowired private DirectChannel inboundMGet; + @Autowired + private DirectChannel inboundMGetRecursive; + @Autowired private SessionFactory sessionFactory; @@ -110,7 +115,9 @@ private void setUpMocksIfNeeded() throws IOException { LsEntry entry4 = mock(LsEntry.class); SftpATTRS attrs4 = mock(SftpATTRS.class); when(entry4.getAttrs()).thenReturn(attrs4); - when(entry4.getFilename()).thenReturn("subSftpSource1.txt"); + // recursion uses a DFA to update the filename to include the subdirectory + new DirectFieldAccessor(entry4).setPropertyValue("filename", "subSftpSource1.txt"); + when(entry4.getFilename()).thenCallRealMethod(); when(session.list("sftpSource/sftpSource1.txt")).thenReturn(new LsEntry[] { entry1 }); @@ -203,4 +210,43 @@ public void testInt2866LocalDirectoryExpressionMGET() { } } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursive() { + String dir = "sftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + assertEquals(3, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subSftpSource")); + + } + + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() { + String dir = "sftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + // should have filtered sftpSource2.txt + assertEquals(2, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subSftpSource")); + + } + } diff --git a/src/reference/docbook/ftp.xml b/src/reference/docbook/ftp.xml index 213865becfb..ca248bcc8b9 100644 --- a/src/reference/docbook/ftp.xml +++ b/src/reference/docbook/ftp.xml @@ -351,6 +351,7 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { -f - do not sort the list -dirs - include directories (excluded by default) -links - include symbolic links (excluded by default) + -R - list the remote directory recursively @@ -366,6 +367,13 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { The remote directory that the ls command acted on is provided in the file_remoteDirectory header. + + When using the recursive option (-R), the fileName includes any subdirectory + elements, representing a relative path to the file (relative to the remote directory). If the -dirs + option is included, each recursed directory is also returned as an element in the list. In this case, + it is recommended that the -1 is not used because you would not be able to determine files Vs. + directories, which is achievable using the FileInfo objects. + get get retrieves a remote file and supports the following option: @@ -399,6 +407,27 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { for the filenames is provided in the file_remoteFile header. + + Notes for when using recursion (<code>-R</code>) + + The pattern is ignored, and * is assumed. By + default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a + FileListFilter; directories in the tree can also be filtered this way. + A FileListFilter can be provided by reference or by filename-pattern + or filename-regex attributes. For example, + filename-regex="(subDir|.*1.txt)" will retrieve all files ending with 1.txt in the + remote directory and the subdirectory subDir. If a subdirectory is filtered, no additional + traversal of that subdirectory is performed. + + + The -dirs option is not allowed (the recursive mget uses the recursive ls to + obtain the directory tree and the directories themselves cannot be included in the list). + + + Typically, you would use the #remoteDirectory variable in the local-directory-expression + so that the remote directory structure is retained locally. + + rm The rm command has no options. diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 163c73b664e..92edb636e9b 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -387,6 +387,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp -f - do not sort the list -dirs - include directories (excluded by default) -links - include symbolic links (excluded by default) + -R - list the remote directory recursively @@ -402,6 +403,13 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp The remote directory that the ls command acted on is provided in the file_remoteDirectory header. + + When using the recursive option (-R), the fileName includes any subdirectory + elements, representing a relative path to the file (relative to the remote directory). If the -dirs + option is included, each recursed directory is also returned as an element in the list. In this case, + it is recommended that the -1 is not used because you would not be able to determine files Vs. + directories, which is achievable using the FileInfo objects. + get get retrieves a remote file and supports the following option: @@ -435,6 +443,27 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp for the filenames is provided in the file_remoteFile header. + + Notes for when using recursion (<code>-R</code>) + + The pattern is ignored, and * is assumed. By + default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a + FileListFilter; directories in the tree can also be filtered this way. + A FileListFilter can be provided by reference or by filename-pattern + or filename-regex attributes. For example, + filename-regex="(subDir|.*1.txt)" will retrieve all files ending with 1.txt in the + remote directory and the subdirectory subDir. If a subdirectory is filtered, no additional + traversal of that subdirectory is performed. + + + The -dirs option is not allowed (the recursive mget uses the recursive ls to + obtain the directory tree and the directories themselves cannot be included in the list). + + + Typically, you would use the #remoteDirectory variable in the local-directory-expression + so that the remote directory structure is retained locally. + + rm The rm command has no options. diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index e4810384a59..e14ed1ffd38 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -238,17 +238,25 @@
FTP, SFTP and FTPS Gateways - The gateways now support the mv command, enabling the renaming of remote - files. - - - The local-filename-generator-expression attribute is now supported, - enabling the naming of local files during transfer. By default, the same - name as the remote file is used. - - - The local-directory-expression attribute is now supported, - enabling the naming of local directories during transfer based on the remote directory. + + + The gateways now support the mv command, enabling the renaming of remote + files. + + + The gateways now support recursive ls and mget commands, enabling + the retrieval of a remote file tree. + + + The local-filename-generator-expression attribute is now supported, + enabling the naming of local files during transfer. By default, the same + name as the remote file is used. + + + The local-directory-expression attribute is now supported, + enabling the naming of local directories during transfer based on the remote directory. + + For more information, see and .