From 69fe200d05a020bbc73c783651110eb35dbb02e1 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Wed, 16 Oct 2013 17:34:06 -0400 Subject: [PATCH 1/6] INT-3172 (S)FTP - Add Recursion Option to LS Cmd When recursing, the returned filenames are relative to the top level directory. --- .../AbstractRemoteFileOutboundGateway.java | 49 ++++++---- .../RemoteFileOutboundGatewayTests.java | 96 +++++++++++++++++-- .../ftp/gateway/FtpOutboundGateway.java | 10 +- .../sftp/gateway/SftpOutboundGateway.java | 10 +- 4 files changed, 141 insertions(+), 24 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index a5c182c4ab5..108f3faf734 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -139,7 +139,11 @@ public static enum Option { /** * Throw an exception if no files returned (mget). */ - EXCEPTION_WHEN_EMPTY("-x"); + EXCEPTION_WHEN_EMPTY("-x"), + /** + * Recursive (ls, mget) + */ + RECURSIVE("-R"); private String option; @@ -398,21 +402,7 @@ private Object doMv(Message requestMessage, Session session) throws IOExce } protected List ls(Session session, String dir) throws IOException { - List lsFiles = new ArrayList(); - F[] files = session.list(dir); - if (!ObjectUtils.isEmpty(files)) { - Collection filteredFiles = this.filterFiles(files); - for (F file : filteredFiles) { - if (file != null) { - if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) { - lsFiles.add(file); - } - } - } - } - else { - return lsFiles; - } + List lsFiles = listFilesInRemoteDir(session, dir, ""); if (!this.options.contains(Option.LINKS)) { purgeLinks(lsFiles); } @@ -441,6 +431,32 @@ protected List ls(Session session, String dir) throws IOException { } } + private List listFilesInRemoteDir(Session session, String directory, String subDirectory) throws IOException { + List lsFiles = new ArrayList(); + F[] files = session.list(directory + subDirectory); + boolean recursing = this.options.contains(Option.RECURSIVE); + if (!ObjectUtils.isEmpty(files)) { + Collection filteredFiles = this.filterFiles(files); + for (F file : filteredFiles) { + String fileName = this.getFilename(file); + if (file != null) { + if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) { + if (recursing && StringUtils.hasText(subDirectory)) { + lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory)); + } + else { + lsFiles.add(file); + } + } + if (recursing && this.isDirectory(file)) { + lsFiles.addAll(listFilesInRemoteDir(session, directory, subDirectory + fileName + File.separator)); + } + } + } + } + return lsFiles; + } + protected final List filterFiles(F[] files) { return (this.filter != null) ? this.filter.filterFiles(files) : Arrays.asList(files); } @@ -630,4 +646,5 @@ private String generateLocalFileName(Message message, String remoteFileName){ abstract protected List> asFileInfoList(Collection files); + abstract protected F enhanceNameWithSubDirectory(F file, String directory); } diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java index 80213429b71..9ff57e88baa 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java @@ -389,9 +389,6 @@ public Object answer(InvocationOnMock invocation) throws Throwable { assertEquals("foo/bar", madeDirs.get(1)); } - /** - * @return - */ public TestLsEntry[] fileList() { TestLsEntry[] files = new TestLsEntry[6]; files[0] = new TestLsEntry("f2", 123, false, false, 1234, "-r--r--r--"); @@ -424,6 +421,83 @@ public void testLs_f() throws Exception { out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); } + public TestLsEntry[] level1List() { + return new TestLsEntry[] { + new TestLsEntry("f1", 123, false, false, 1234, "-r--r--r--"), + new TestLsEntry("d1", 0, true, false, 12345, "drw-r--r--"), + new TestLsEntry("f2", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + public TestLsEntry[] level2List() { + return new TestLsEntry[] { + new TestLsEntry("d2", 0, true, false, 12345, "drw-r--r--"), + new TestLsEntry("f3", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + public TestLsEntry[] level3List() { + return new TestLsEntry[] { + new TestLsEntry("f4", 12345, false, false, 123456, "-rw-r--r--") + }; + } + + @Test + public void testLs_f_R() throws Exception { + SessionFactory sessionFactory = mock(SessionFactory.class); + Session session = mock(Session.class); + TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway + (sessionFactory, "ls", "payload"); + gw.setOptions("-f -R"); + gw.afterPropertiesSet(); + when(sessionFactory.getSession()).thenReturn(session); + TestLsEntry[] level1 = level1List(); + TestLsEntry[] level2 = level2List(); + TestLsEntry[] level3 = level3List(); + when(session.list("testremote/x/")).thenReturn(level1); + when(session.list("testremote/x/d1/")).thenReturn(level2); + when(session.list("testremote/x/d1/d2/")).thenReturn(level3); + @SuppressWarnings("unchecked") + Message> out = (Message>) gw + .handleRequestMessage(new GenericMessage("testremote/x")); + assertEquals(4, out.getPayload().size()); + assertEquals("f1", out.getPayload().get(0).getFilename()); + assertEquals("d1/d2/f4", out.getPayload().get(1).getFilename()); + assertEquals("d1/f3", out.getPayload().get(2).getFilename()); + assertEquals("f2", out.getPayload().get(3).getFilename()); + assertEquals("testremote/x/", + out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); + } + + @Test + public void testLs_f_R_dirs() throws Exception { + SessionFactory sessionFactory = mock(SessionFactory.class); + Session session = mock(Session.class); + TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway + (sessionFactory, "ls", "payload"); + gw.setOptions("-f -R -dirs"); + gw.afterPropertiesSet(); + when(sessionFactory.getSession()).thenReturn(session); + TestLsEntry[] level1 = level1List(); + TestLsEntry[] level2 = level2List(); + TestLsEntry[] level3 = level3List(); + when(session.list("testremote/x/")).thenReturn(level1); + when(session.list("testremote/x/d1/")).thenReturn(level2); + when(session.list("testremote/x/d1/d2/")).thenReturn(level3); + @SuppressWarnings("unchecked") + Message> out = (Message>) gw + .handleRequestMessage(new GenericMessage("testremote/x")); + assertEquals(6, out.getPayload().size()); + assertEquals("f1", out.getPayload().get(0).getFilename()); + assertEquals("d1", out.getPayload().get(1).getFilename()); + assertEquals("d1/d2", out.getPayload().get(2).getFilename()); + assertEquals("d1/d2/f4", out.getPayload().get(3).getFilename()); + assertEquals("d1/f3", out.getPayload().get(4).getFilename()); + assertEquals("f2", out.getPayload().get(5).getFilename()); + assertEquals("testremote/x/", + out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY)); + } + @Test public void testLs_None() throws Exception { SessionFactory sessionFactory = mock(SessionFactory.class); @@ -786,18 +860,24 @@ protected List> asFileInfoList( return new ArrayList>(files); } + @Override + protected TestLsEntry enhanceNameWithSubDirectory(TestLsEntry file, String directory) { + file.setFilename(directory + file.getFilename()); + return file; + } + } class TestLsEntry extends AbstractFileInfo { - private final String filename; - private final int size; + private volatile String filename; + private final long size; private final boolean dir; private final boolean link; private final long modified; private final String permissions; - public TestLsEntry(String filename, int size, boolean dir, boolean link, + public TestLsEntry(String filename, long size, boolean dir, boolean link, long modified, String permissions) { this.filename = filename; this.size = size; @@ -835,6 +915,10 @@ public TestLsEntry getFileInfo() { return this; } + public void setFilename(String filename) { + this.filename = filename; + } + } class TestPatternFilter extends AbstractSimplePatternFileListFilter{ diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 75a1f8f8708..172fb1b56c8 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -21,6 +21,7 @@ import java.util.List; import org.apache.commons.net.ftp.FTPFile; + import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; @@ -28,7 +29,7 @@ /** * Outbound Gateway for performing remote file operations via FTP/FTPS. - * + * * @author Gary Russell * @since 2.1 */ @@ -69,4 +70,11 @@ protected List> asFileInfoList(Collection fil } + @Override + protected FTPFile enhanceNameWithSubDirectory(FTPFile file, String directory) { + file.setName(directory + file.getName()); + return file; + } + + } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java index cdb3755be94..cc40084443d 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.List; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.integration.file.remote.AbstractFileInfo; import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; import org.springframework.integration.file.remote.session.SessionFactory; @@ -29,7 +30,7 @@ /** * Outbound Gateway for performing remote file operations via SFTP. - * + * * @author Gary Russell * @since 2.1 */ @@ -73,4 +74,11 @@ protected long getModified(LsEntry file) { return ((long)file.getAttrs().getMTime()) * 1000; } + @Override + protected LsEntry enhanceNameWithSubDirectory(LsEntry file, String directory) { + DirectFieldAccessor accessor = new DirectFieldAccessor(file); + accessor.setPropertyValue("filename", directory + file.getFilename()); + return file; + } + } From 461371f966a815f8834e0a42b349a5b6140b6c20 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Wed, 16 Oct 2013 18:32:21 -0400 Subject: [PATCH 2/6] INT-3172 (S)FTP - Add Recursion to MGET Command https://jira.springsource.org/browse/INT-3172 --- .../AbstractRemoteFileOutboundGateway.java | 47 +++++++++++++++++++ .../RemoteFileOutboundGatewayTests.java | 5 ++ .../ftp/gateway/FtpOutboundGateway.java | 5 ++ .../FtpServerOutboundTests-context.xml | 11 +++++ .../ftp/outbound/FtpServerOutboundTests.java | 21 +++++++++ 5 files changed, 89 insertions(+) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index 108f3faf734..38fbe1d290d 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -309,6 +309,11 @@ protected void onInit() { } } } + if (Command.MGET.equals(this.command)) { + Assert.isTrue(!(this.options.contains(Option.SUBDIRS)), + "Cannot use " + Option.SUBDIRS.toString() + " when using 'mget' use " + Option.RECURSIVE.toString() + + " to obtain files in subdirectories"); + } if (this.getBeanFactory() != null) { this.fileNameProcessor.setBeanFactory(this.getBeanFactory()); this.renameProcessor.setBeanFactory(this.getBeanFactory()); @@ -539,6 +544,22 @@ protected File get(Message message, Session session, String remoteDir, Str protected List mGet(Message message, Session session, String remoteDirectory, String remoteFilename) throws IOException { + if (this.options.contains(Option.RECURSIVE)) { + if (logger.isWarnEnabled() && !("*".equals(remoteFilename))) { + logger.warn("File name pattern must be '*' when using recursion"); + } + if (this.options.contains(Option.NAME_ONLY)) { + this.options.remove(Option.NAME_ONLY); + } + return mGetWithRecursion(message, session, remoteDirectory, remoteFilename); + } + else { + return mGetWithoutRecursion(message, session, remoteDirectory, remoteFilename); + } + } + + private List mGetWithoutRecursion(Message message, Session session, String remoteDirectory, + String remoteFilename) throws IOException { String path = this.generateFullPath(remoteDirectory, remoteFilename); String[] fileNames = session.listNames(path); if (fileNames == null) { @@ -565,6 +586,30 @@ protected List mGet(Message message, Session session, String remoteD return files; } + private List mGetWithRecursion(Message message, Session session, String remoteDirectory, + String remoteFilename) throws IOException { + List files = new ArrayList(); + @SuppressWarnings("unchecked") + List> fileNames = (List>) this.ls(session, remoteDirectory); + if (fileNames.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) { + throw new MessagingException("No files found at " + remoteDirectory + + " with pattern " + remoteFilename); + } + for (AbstractFileInfo lsEntry : fileNames) { + String fullFileName = remoteDirectory + this.getFilename(lsEntry); + /* + * With recursion, the filename might contain subdirectory information + * normalize each file separately. + */ + String fileName = this.getRemoteFilename(fullFileName); + String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName); + File file = this.get(message, session, actualRemoteDirectory, + fullFileName, fileName, false); + files.add(file); + } + return files; + } + private String getRemoteDirectory(String remoteFilePath, String remoteFilename) { String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf(remoteFilename)); if (remoteDir.length() == 0) { @@ -642,6 +687,8 @@ private String generateLocalFileName(Message message, String remoteFileName){ abstract protected String getFilename(F file); + abstract protected String getFilename(AbstractFileInfo file); + abstract protected long getModified(F file); abstract protected List> asFileInfoList(Collection files); diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java index 9ff57e88baa..c69a9580eae 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java @@ -849,6 +849,11 @@ protected String getFilename(TestLsEntry file) { return file.getFilename(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected long getModified(TestLsEntry file) { return file.getModified(); diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 172fb1b56c8..079b811dac9 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -55,6 +55,11 @@ protected String getFilename(FTPFile file) { return file.getName(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected long getModified(FTPFile file) { return file.getTimestamp().getTimeInMillis(); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index 91feb884b4f..c936f6b2330 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -50,5 +50,16 @@ local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> + + + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 61fec74f09e..5ce1f4fa711 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.ftp.outbound; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -61,6 +62,9 @@ public class FtpServerOutboundTests { @Autowired private DirectChannel inboundMGet; + @Autowired + private DirectChannel inboundMGetRecursive; + @Before public void setup() { FtpServerRule.recursiveDelete(FTP_SERVER.getTargetLocalDirectory()); @@ -125,6 +129,23 @@ public void testInt2866LocalDirectoryExpressionMGET() { } } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursive() { + String dir = "ftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + assertEquals(3, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + + } + public static String localDirectory() { return FTP_SERVER.getTargetLocalDirectory().getAbsolutePath() + File.separator; } From 834cb1851436ed451f991784329b57599421b72a Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 17 Oct 2013 10:17:48 -0400 Subject: [PATCH 3/6] INT-3172 Polishing - PR Comments - Add Tests for SFTP (tested with Mock and real SSH) - Enhance test to ensure subdir is in 3rd file retrieved - Exclude "special" directories ('.' and '..') from recursion --- .../AbstractRemoteFileOutboundGateway.java | 17 +++++++++-- .../ftp/gateway/FtpOutboundGateway.java | 2 +- .../ftp/outbound/FtpServerOutboundTests.java | 2 ++ .../sftp/gateway/SftpOutboundGateway.java | 7 ++++- .../SftpServerOutboundTests-context.xml | 11 ++++++++ .../outbound/SftpServerOutboundTests.java | 28 ++++++++++++++++++- 6 files changed, 62 insertions(+), 5 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index 38fbe1d290d..ae772685602 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -66,22 +66,27 @@ public abstract class AbstractRemoteFileOutboundGateway extends AbstractReply * Enumeration of commands supported by the gateways. */ public static enum Command { + /** * List remote files. */ LS("ls"), + /** * Retrieve a remote file. */ GET("get"), + /** * Remove a remote file (path - including wildcards). */ RM("rm"), + /** * Retrieve multiple files matching a wildcard path. */ MGET("mget"), + /** * Move (rename) a remote file. */ @@ -112,34 +117,42 @@ public static Command toCommand(String cmd) { * */ public static enum Option { + /** * Don't return full file information; just the name (ls). */ NAME_ONLY("-1"), + /** - * Include directories {@code .} and {@code ..} in the results (ls). + * Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls). */ ALL("-a"), + /** * Do not sort the results (ls with NAME_ONLY). */ NOSORT("-f"), + /** * Include directories in the results (ls). */ SUBDIRS("-dirs"), + /** * Include links in the results (ls). */ LINKS("-links"), + /** * Preserve the server timestamp (get, mget). */ PRESERVE_TIMESTAMP("-P"), + /** * Throw an exception if no files returned (mget). */ EXCEPTION_WHEN_EMPTY("-x"), + /** * Recursive (ls, mget) */ @@ -453,7 +466,7 @@ private List listFilesInRemoteDir(Session session, String directory, Strin lsFiles.add(file); } } - if (recursing && this.isDirectory(file)) { + if (recursing && this.isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) { lsFiles.addAll(listFilesInRemoteDir(session, directory, subDirectory + fileName + File.separator)); } } diff --git a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java index 079b811dac9..b55c612f5e4 100644 --- a/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java +++ b/spring-integration-ftp/src/main/java/org/springframework/integration/ftp/gateway/FtpOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 5ce1f4fa711..4d81cbc9b2a 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -143,6 +143,8 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() { assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), Matchers.containsString(dir)); } + assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subFtpSource")); } diff --git a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java index cc40084443d..72c7db47504 100644 --- a/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java +++ b/spring-integration-sftp/src/main/java/org/springframework/integration/sftp/gateway/SftpOutboundGateway.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2011 the original author or authors. + * Copyright 2002-2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -60,6 +60,11 @@ protected String getFilename(LsEntry file) { return file.getFilename(); } + @Override + protected String getFilename(AbstractFileInfo file) { + return file.getFilename(); + } + @Override protected List> asFileInfoList(Collection files) { List> canonicalFiles = new ArrayList>(); diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml index 5c259ebbcf9..b26b052030e 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml @@ -41,6 +41,17 @@ local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> + + + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java index 0a964c5de08..8e5ba63842a 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java @@ -16,6 +16,7 @@ package org.springframework.integration.sftp.outbound; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -32,6 +33,7 @@ import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.DirectFieldAccessor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.Message; import org.springframework.integration.channel.DirectChannel; @@ -80,6 +82,9 @@ public class SftpServerOutboundTests { @Autowired private DirectChannel inboundMGet; + @Autowired + private DirectChannel inboundMGetRecursive; + @Autowired private SessionFactory sessionFactory; @@ -110,7 +115,9 @@ private void setUpMocksIfNeeded() throws IOException { LsEntry entry4 = mock(LsEntry.class); SftpATTRS attrs4 = mock(SftpATTRS.class); when(entry4.getAttrs()).thenReturn(attrs4); - when(entry4.getFilename()).thenReturn("subSftpSource1.txt"); + // recursion uses a DFA to update the filename to include the subdirectory + new DirectFieldAccessor(entry4).setPropertyValue("filename", "subSftpSource1.txt"); + when(entry4.getFilename()).thenCallRealMethod(); when(session.list("sftpSource/sftpSource1.txt")).thenReturn(new LsEntry[] { entry1 }); @@ -203,4 +210,23 @@ public void testInt2866LocalDirectoryExpressionMGET() { } } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursive() { + String dir = "sftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + assertEquals(3, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(2).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subSftpSource")); + + } + } From 8277c80655cdd8cb151091e9b232b4952f9a0477 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 17 Oct 2013 10:45:50 -0400 Subject: [PATCH 4/6] INT-3172 Make FtpServerRule Safe For Concurrency Previously the port was static. - Make the port an instance variable - Channel the `@ ClassRule` to a `@ Rule` - Register an appropriate DefaultFtpSessionFactory bean declaration --- .../integration/ftp/FtpServerRule.java | 26 +++++++++++++++++-- .../FtpServerOutboundTests-context.xml | 15 ++++------- .../ftp/outbound/FtpServerOutboundTests.java | 16 +++++------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java index 260d34d3c27..17b20799df6 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java @@ -35,15 +35,23 @@ import org.junit.rules.ExternalResource; import org.junit.rules.TemporaryFolder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.integration.ftp.session.DefaultFtpSessionFactory; import org.springframework.integration.test.util.SocketUtils; /** + * Embedded FTP Server for test cases; exposes an associated session factory + * as a @Bean. + * * @author Artem Bilan + * @author Gary Russell * @since 3.0 */ +@Configuration public class FtpServerRule extends ExternalResource { - public static int FTP_PORT = SocketUtils.findAvailableServerSocket(); + private final int ftpPort = SocketUtils.findAvailableServerSocket(); private final TemporaryFolder ftpFolder; @@ -124,6 +132,20 @@ public File getTargetLocalDirectory() { return targetLocalDirectory; } + public String getTargetLocalDirectoryName() { + return targetLocalDirectory.getAbsolutePath() + File.separator; + } + + @Bean + public DefaultFtpSessionFactory ftpSessionFactory() { + DefaultFtpSessionFactory factory = new DefaultFtpSessionFactory(); + factory.setHost("localhost"); + factory.setPort(this.ftpPort); + factory.setUsername("foo"); + factory.setPassword("foo"); + return factory; + } + @Override protected void before() throws Throwable { this.ftpFolder.create(); @@ -133,7 +155,7 @@ protected void before() throws Throwable { serverFactory.setUserManager(new TestUserManager(this.ftpRootFolder.getAbsolutePath())); ListenerFactory factory = new ListenerFactory(); - factory.setPort(FTP_PORT); + factory.setPort(ftpPort); serverFactory.addListener("default", factory.createListener()); server = serverFactory.createServer(); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index c936f6b2330..142a7200f17 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -8,15 +8,10 @@ http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> - - - - - + + - - @@ -27,7 +22,7 @@ request-channel="inboundGet" command="get" expression="payload" - local-directory-expression="#localDir() + #remoteDirectory.toUpperCase()" + local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory.toUpperCase()" local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> @@ -46,7 +41,7 @@ request-channel="inboundMGet" command="mget" expression="payload" - local-directory-expression="#localDir() + #remoteDirectory" + local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory" local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> @@ -57,7 +52,7 @@ command="mget" expression="payload" command-options="-R" - local-directory-expression="#localDir() + #remoteDirectory" + local-directory-expression="@ftpServer.targetLocalDirectoryName + #remoteDirectory" local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index 4d81cbc9b2a..f7e2fbf5053 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -26,7 +26,7 @@ import org.hamcrest.Matchers; import org.junit.Before; -import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -47,8 +47,9 @@ @RunWith(SpringJUnit4ClassRunner.class) public class FtpServerOutboundTests { - @ClassRule - public static final FtpServerRule FTP_SERVER = new FtpServerRule(FtpServerOutboundTests.class.getSimpleName()); + @Rule + @Autowired + public FtpServerRule ftpServer; @Autowired private PollableChannel output; @@ -67,8 +68,8 @@ public class FtpServerOutboundTests { @Before public void setup() { - FtpServerRule.recursiveDelete(FTP_SERVER.getTargetLocalDirectory()); - FtpServerRule.recursiveDelete(FTP_SERVER.getTargetFtpDirectory()); + FtpServerRule.recursiveDelete(ftpServer.getTargetLocalDirectory()); + FtpServerRule.recursiveDelete(ftpServer.getTargetFtpDirectory()); } @Test @@ -148,9 +149,4 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() { } - public static String localDirectory() { - return FTP_SERVER.getTargetLocalDirectory().getAbsolutePath() + File.separator; - } - - } From 9c13d25b41b61e437e29d5d115eaa7b459e1102f Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 17 Oct 2013 13:07:23 -0400 Subject: [PATCH 5/6] INT-3172 Polishing; Docs - Rename FtpServerRule to TestFtpServer - Remove restriction on filters for MGET - Add test for filtered MGET - Reference docs, what's new --- .../AbstractRemoteFileOutboundGateway.java | 2 +- .../{FtpServerRule.java => TesFtpServer.java} | 16 +++++---- .../FtpServerOutboundTests-context.xml | 13 +++++++- .../ftp/outbound/FtpServerOutboundTests.java | 33 +++++++++++++++---- .../SftpServerOutboundTests-context.xml | 12 +++++++ .../outbound/SftpServerOutboundTests.java | 20 +++++++++++ src/reference/docbook/ftp.xml | 29 ++++++++++++++++ src/reference/docbook/sftp.xml | 29 ++++++++++++++++ src/reference/docbook/whats-new.xml | 30 ++++++++++------- 9 files changed, 158 insertions(+), 26 deletions(-) rename spring-integration-ftp/src/test/java/org/springframework/integration/ftp/{FtpServerRule.java => TesFtpServer.java} (96%) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index ae772685602..082bda057e0 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -289,7 +289,7 @@ public void setLocalFilenameGeneratorExpression(Expression localFilenameGenerato protected void onInit() { super.onInit(); Assert.notNull(this.command, "command must not be null"); - if (Command.RM.equals(this.command) || Command.MGET.equals(this.command) || + if (Command.RM.equals(this.command) || Command.GET.equals(this.command)) { Assert.isNull(this.filter, "Filters are not supported with the rm, get, and mget commands"); } diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java similarity index 96% rename from spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java rename to spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java index 17b20799df6..9d059f3f103 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/FtpServerRule.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/TesFtpServer.java @@ -20,6 +20,9 @@ import java.io.IOException; import java.util.Arrays; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + import org.apache.ftpserver.FtpServer; import org.apache.ftpserver.FtpServerFactory; import org.apache.ftpserver.ftplet.Authentication; @@ -32,7 +35,6 @@ import org.apache.ftpserver.usermanager.impl.ConcurrentLoginPermission; import org.apache.ftpserver.usermanager.impl.TransferRatePermission; import org.apache.ftpserver.usermanager.impl.WritePermission; -import org.junit.rules.ExternalResource; import org.junit.rules.TemporaryFolder; import org.springframework.context.annotation.Bean; @@ -49,7 +51,7 @@ * @since 3.0 */ @Configuration -public class FtpServerRule extends ExternalResource { +public class TesFtpServer { private final int ftpPort = SocketUtils.findAvailableServerSocket(); @@ -69,7 +71,7 @@ public class FtpServerRule extends ExternalResource { private volatile FtpServer server; - public FtpServerRule(final String root) { + public TesFtpServer(final String root) { this.ftpFolder = new TemporaryFolder() { @Override @@ -146,8 +148,8 @@ public DefaultFtpSessionFactory ftpSessionFactory() { return factory; } - @Override - protected void before() throws Throwable { + @PostConstruct + public void before() throws Throwable { this.ftpFolder.create(); this.localFolder.create(); @@ -163,8 +165,8 @@ protected void before() throws Throwable { } - @Override - protected void after() { + @PreDestroy + public void after() { this.server.stop(); this.ftpFolder.delete(); this.localFolder.delete(); diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml index 142a7200f17..9fe16dd9acc 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml @@ -8,7 +8,7 @@ http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> - + @@ -56,5 +56,16 @@ local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> + + + diff --git a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java index f7e2fbf5053..0519eac0d52 100644 --- a/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java +++ b/spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java @@ -26,7 +26,6 @@ import org.hamcrest.Matchers; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,7 +33,7 @@ import org.springframework.integration.Message; import org.springframework.integration.channel.DirectChannel; import org.springframework.integration.core.PollableChannel; -import org.springframework.integration.ftp.FtpServerRule; +import org.springframework.integration.ftp.TesFtpServer; import org.springframework.integration.message.GenericMessage; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -47,9 +46,8 @@ @RunWith(SpringJUnit4ClassRunner.class) public class FtpServerOutboundTests { - @Rule @Autowired - public FtpServerRule ftpServer; + public TesFtpServer ftpServer; @Autowired private PollableChannel output; @@ -66,10 +64,13 @@ public class FtpServerOutboundTests { @Autowired private DirectChannel inboundMGetRecursive; + @Autowired + private DirectChannel inboundMGetRecursiveFiltered; + @Before public void setup() { - FtpServerRule.recursiveDelete(ftpServer.getTargetLocalDirectory()); - FtpServerRule.recursiveDelete(ftpServer.getTargetFtpDirectory()); + TesFtpServer.recursiveDelete(ftpServer.getTargetLocalDirectory()); + TesFtpServer.recursiveDelete(ftpServer.getTargetFtpDirectory()); } @Test @@ -149,4 +150,24 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() { } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() { + String dir = "ftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + // should have filtered ftpSource2.txt + assertEquals(2, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subFtpSource")); + + } + } diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml index b26b052030e..5a530a9517d 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml @@ -52,6 +52,18 @@ local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')" reply-channel="output"/> + + + + diff --git a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java index 8e5ba63842a..6860bdb8875 100644 --- a/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java +++ b/spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java @@ -229,4 +229,24 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() { } + @Test + @SuppressWarnings("unchecked") + public void testInt3172LocalDirectoryExpressionMGETRecursiveFiltered() { + String dir = "sftpSource/"; + this.inboundMGetRecursive.send(new GenericMessage(dir + "*")); + Message result = this.output.receive(1000); + assertNotNull(result); + List localFiles = (List) result.getPayload(); + // should have filtered sftpSource2.txt + assertEquals(2, localFiles.size()); + + for (File file : localFiles) { + assertThat(file.getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir)); + } + assertThat(localFiles.get(1).getPath().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/"), + Matchers.containsString(dir + "subSftpSource")); + + } + } diff --git a/src/reference/docbook/ftp.xml b/src/reference/docbook/ftp.xml index 213865becfb..613300fd799 100644 --- a/src/reference/docbook/ftp.xml +++ b/src/reference/docbook/ftp.xml @@ -351,6 +351,7 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { -f - do not sort the list -dirs - include directories (excluded by default) -links - include symbolic links (excluded by default) + -R - list the remote directory recursively @@ -366,6 +367,13 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { The remote directory that the ls command acted on is provided in the file_remoteDirectory header. + + When using the recursive option (-R), the fileName includes any subdirectory + elements, representing a relative path to the file (relative to the remoe directory). If the -dirs + option is included, each recursed directory is also returned as an element in the list. In this case, + it is recommended that the -1 is not used because you would not be able to determine files Vs. + directories, which is achievable using the FileInfo objects. + get get retrieves a remote file and supports the following option: @@ -399,6 +407,27 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { for the filenames is provided in the file_remoteFile header. + + Notes for when using recursion (<code>-R</code>) + + The pattern is ignored, and * is assumed. By + default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a + FileListFilter; directories in the tree can also be filtered this way. + A FileListFilter can be provided by reference or by filename-pattern + or filename-regex attributes. For example, + filename-regex="(subDir|.*1.txt)" will retrieve all files ending with 1.txt in the + remote directory and the subdirectory subDir. If a subdirectory is filtered, no additional + traversal of that subdirectory is performed. + + + The -dirs option is not allowed (the recursive mget uses the recursive ls to + obtain the directory tree and the directories themselves cannot be included in the list). + + + Typically, you would use the #remoteDirectory variable in the local-directory-expression + so that the remote directory structure is retained locally. + + rm The rm command has no options. diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 163c73b664e..26827eee7cf 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -387,6 +387,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp -f - do not sort the list -dirs - include directories (excluded by default) -links - include symbolic links (excluded by default) + -R - list the remote directory recursively @@ -402,6 +403,13 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp The remote directory that the ls command acted on is provided in the file_remoteDirectory header. + + When using the recursive option (-R), the fileName includes any subdirectory + elements, representing a relative path to the file (relative to the remoe directory). If the -dirs + option is included, each recursed directory is also returned as an element in the list. In this case, + it is recommended that the -1 is not used because you would not be able to determine files Vs. + directories, which is achievable using the FileInfo objects. + get get retrieves a remote file and supports the following option: @@ -435,6 +443,27 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp for the filenames is provided in the file_remoteFile header. + + Notes for when using recursion (<code>-R</code>) + + The pattern is ignored, and * is assumed. By + default, the entire remote tree is retrieved. However, files in the tree can be filtered, by providing a + FileListFilter; directories in the tree can also be filtered this way. + A FileListFilter can be provided by reference or by filename-pattern + or filename-regex attributes. For example, + filename-regex="(subDir|.*1.txt)" will retrieve all files ending with 1.txt in the + remote directory and the subdirectory subDir. If a subdirectory is filtered, no additional + traversal of that subdirectory is performed. + + + The -dirs option is not allowed (the recursive mget uses the recursive ls to + obtain the directory tree and the directories themselves cannot be included in the list). + + + Typically, you would use the #remoteDirectory variable in the local-directory-expression + so that the remote directory structure is retained locally. + + rm The rm command has no options. diff --git a/src/reference/docbook/whats-new.xml b/src/reference/docbook/whats-new.xml index e4810384a59..e14ed1ffd38 100644 --- a/src/reference/docbook/whats-new.xml +++ b/src/reference/docbook/whats-new.xml @@ -238,17 +238,25 @@
FTP, SFTP and FTPS Gateways - The gateways now support the mv command, enabling the renaming of remote - files. - - - The local-filename-generator-expression attribute is now supported, - enabling the naming of local files during transfer. By default, the same - name as the remote file is used. - - - The local-directory-expression attribute is now supported, - enabling the naming of local directories during transfer based on the remote directory. + + + The gateways now support the mv command, enabling the renaming of remote + files. + + + The gateways now support recursive ls and mget commands, enabling + the retrieval of a remote file tree. + + + The local-filename-generator-expression attribute is now supported, + enabling the naming of local files during transfer. By default, the same + name as the remote file is used. + + + The local-directory-expression attribute is now supported, + enabling the naming of local directories during transfer based on the remote directory. + + For more information, see and . From 75be1741e7ddf90ac9b1f84b1de8d01380eb470e Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 17 Oct 2013 13:30:28 -0400 Subject: [PATCH 6/6] INT-3172 More Polish - Fix type in reference - Fix error message now that filter is allowed with MGET - Remove test for filter with MGET (now valid) --- .../AbstractRemoteFileOutboundGateway.java | 2 +- .../gateway/RemoteFileOutboundGatewayTests.java | 15 --------------- src/reference/docbook/ftp.xml | 2 +- src/reference/docbook/sftp.xml | 2 +- 4 files changed, 3 insertions(+), 18 deletions(-) diff --git a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java index 082bda057e0..d37f34de145 100644 --- a/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java +++ b/spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java @@ -291,7 +291,7 @@ protected void onInit() { Assert.notNull(this.command, "command must not be null"); if (Command.RM.equals(this.command) || Command.GET.equals(this.command)) { - Assert.isNull(this.filter, "Filters are not supported with the rm, get, and mget commands"); + Assert.isNull(this.filter, "Filters are not supported with the rm and get commands"); } if (Command.GET.equals(this.command) || Command.MGET.equals(this.command)) { diff --git a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java index c69a9580eae..eca3f97fa3e 100644 --- a/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java +++ b/spring-integration-file/src/test/java/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java @@ -86,21 +86,6 @@ public void testBadFilterGet() throws Exception { } } - @Test - public void testBadFilterMGet() throws Exception { - SessionFactory sessionFactory = mock(SessionFactory.class); - TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway - (sessionFactory, "mget", "payload"); - gw.setFilter(new TestPatternFilter("")); - try { - gw.onInit(); - fail("Exception expected"); - } - catch (IllegalArgumentException e) { - assertTrue(e.getMessage().startsWith("Filters are not supported")); - } - } - @Test public void testBadFilterRm() throws Exception { SessionFactory sessionFactory = mock(SessionFactory.class); diff --git a/src/reference/docbook/ftp.xml b/src/reference/docbook/ftp.xml index 613300fd799..ca248bcc8b9 100644 --- a/src/reference/docbook/ftp.xml +++ b/src/reference/docbook/ftp.xml @@ -369,7 +369,7 @@ protected void postProcessClientBeforeConnect(T client) throws IOException { When using the recursive option (-R), the fileName includes any subdirectory - elements, representing a relative path to the file (relative to the remoe directory). If the -dirs + elements, representing a relative path to the file (relative to the remote directory). If the -dirs option is included, each recursed directory is also returned as an element in the list. In this case, it is recommended that the -1 is not used because you would not be able to determine files Vs. directories, which is achievable using the FileInfo objects. diff --git a/src/reference/docbook/sftp.xml b/src/reference/docbook/sftp.xml index 26827eee7cf..92edb636e9b 100644 --- a/src/reference/docbook/sftp.xml +++ b/src/reference/docbook/sftp.xml @@ -405,7 +405,7 @@ xsi:schemaLocation="http://www.springframework.org/schema/integration/sftp When using the recursive option (-R), the fileName includes any subdirectory - elements, representing a relative path to the file (relative to the remoe directory). If the -dirs + elements, representing a relative path to the file (relative to the remote directory). If the -dirs option is included, each recursed directory is also returned as an element in the list. In this case, it is recommended that the -1 is not used because you would not be able to determine files Vs. directories, which is achievable using the FileInfo objects.