Skip to content

INT-3172 Add Recursion to LS/MGET #927

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

INT-3172 Add Recursion to LS/MGET #927

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
69fe200
INT-3172 (S)FTP - Add Recursion Option to LS Cmd
garyrussell Oct 16, 2013
461371f
INT-3172 (S)FTP - Add Recursion to MGET Command
garyrussell Oct 16, 2013
834cb18
INT-3172 Polishing
garyrussell Oct 17, 2013
8277c80
INT-3172 Make FtpServerRule Safe For Concurrency
garyrussell Oct 17, 2013
9c13d25
INT-3172 Polishing; Docs
garyrussell Oct 17, 2013
75be174
INT-3172 More Polish
garyrussell Oct 17, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 96 additions & 19 deletions ...rg/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,27 @@ public abstract class AbstractRemoteFileOutboundGateway<F> extends AbstractReply
* Enumeration of commands supported by the gateways.
*/
public static enum Command {

/**
* List remote files.
*/
LS("ls"),

/**
* Retrieve a remote file.
*/
GET("get"),

/**
* Remove a remote file (path - including wildcards).
*/
RM("rm"),

/**
* Retrieve multiple files matching a wildcard path.
*/
MGET("mget"),

/**
* Move (rename) a remote file.
*/
Expand Down Expand Up @@ -112,34 +117,46 @@ public static Command toCommand(String cmd) {
*
*/
public static enum Option {

/**
* Don't return full file information; just the name (ls).
*/
NAME_ONLY("-1"),

/**
* Include directories {@code .} and {@code ..} in the results (ls).
* Include files beginning with {@code .}, including directories {@code .} and {@code ..} in the results (ls).
*/
ALL("-a"),

/**
* Do not sort the results (ls with NAME_ONLY).
*/
NOSORT("-f"),

/**
* Include directories in the results (ls).
*/
SUBDIRS("-dirs"),

/**
* Include links in the results (ls).
*/
LINKS("-links"),

/**
* Preserve the server timestamp (get, mget).
*/
PRESERVE_TIMESTAMP("-P"),

/**
* Throw an exception if no files returned (mget).
*/
EXCEPTION_WHEN_EMPTY("-x");
EXCEPTION_WHEN_EMPTY("-x"),

/**
* Recursive (ls, mget)
*/
RECURSIVE("-R");

private String option;

Expand Down Expand Up @@ -272,9 +289,9 @@ public void setLocalFilenameGeneratorExpression(Expression localFilenameGenerato
protected void onInit() {
super.onInit();
Assert.notNull(this.command, "command must not be null");
if (Command.RM.equals(this.command) || Command.MGET.equals(this.command) ||
if (Command.RM.equals(this.command) ||
Command.GET.equals(this.command)) {
Assert.isNull(this.filter, "Filters are not supported with the rm, get, and mget commands");
Assert.isNull(this.filter, "Filters are not supported with the rm and get commands");
}
if (Command.GET.equals(this.command)
|| Command.MGET.equals(this.command)) {
Expand Down Expand Up @@ -305,6 +322,11 @@ protected void onInit() {
}
}
}
if (Command.MGET.equals(this.command)) {
Assert.isTrue(!(this.options.contains(Option.SUBDIRS)),
"Cannot use " + Option.SUBDIRS.toString() + " when using 'mget' use " + Option.RECURSIVE.toString() +
" to obtain files in subdirectories");
}
if (this.getBeanFactory() != null) {
this.fileNameProcessor.setBeanFactory(this.getBeanFactory());
this.renameProcessor.setBeanFactory(this.getBeanFactory());
Expand Down Expand Up @@ -398,21 +420,7 @@ private Object doMv(Message<?> requestMessage, Session<F> session) throws IOExce
}

protected List<?> ls(Session<F> session, String dir) throws IOException {
List<F> lsFiles = new ArrayList<F>();
F[] files = session.list(dir);
if (!ObjectUtils.isEmpty(files)) {
Collection<F> filteredFiles = this.filterFiles(files);
for (F file : filteredFiles) {
if (file != null) {
if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) {
lsFiles.add(file);
}
}
}
}
else {
return lsFiles;
}
List<F> lsFiles = listFilesInRemoteDir(session, dir, "");
if (!this.options.contains(Option.LINKS)) {
purgeLinks(lsFiles);
}
Expand Down Expand Up @@ -441,6 +449,32 @@ protected List<?> ls(Session<F> session, String dir) throws IOException {
}
}

private List<F> listFilesInRemoteDir(Session<F> session, String directory, String subDirectory) throws IOException {
List<F> lsFiles = new ArrayList<F>();
F[] files = session.list(directory + subDirectory);
boolean recursing = this.options.contains(Option.RECURSIVE);
if (!ObjectUtils.isEmpty(files)) {
Collection<F> filteredFiles = this.filterFiles(files);
for (F file : filteredFiles) {
String fileName = this.getFilename(file);
if (file != null) {
if (this.options.contains(Option.SUBDIRS) || !this.isDirectory(file)) {
if (recursing && StringUtils.hasText(subDirectory)) {
lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory));
}
else {
lsFiles.add(file);
}
}
if (recursing && this.isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) {
lsFiles.addAll(listFilesInRemoteDir(session, directory, subDirectory + fileName + File.separator));
}
}
}
}
return lsFiles;
}

protected final List<F> filterFiles(F[] files) {
return (this.filter != null) ? this.filter.filterFiles(files) : Arrays.asList(files);
}
Expand Down Expand Up @@ -523,6 +557,22 @@ protected File get(Message<?> message, Session<F> session, String remoteDir, Str

protected List<File> mGet(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
if (this.options.contains(Option.RECURSIVE)) {
if (logger.isWarnEnabled() && !("*".equals(remoteFilename))) {
logger.warn("File name pattern must be '*' when using recursion");
}
if (this.options.contains(Option.NAME_ONLY)) {
this.options.remove(Option.NAME_ONLY);
}
return mGetWithRecursion(message, session, remoteDirectory, remoteFilename);
}
else {
return mGetWithoutRecursion(message, session, remoteDirectory, remoteFilename);
}
}

private List<File> mGetWithoutRecursion(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
String path = this.generateFullPath(remoteDirectory, remoteFilename);
String[] fileNames = session.listNames(path);
if (fileNames == null) {
Expand All @@ -549,6 +599,30 @@ protected List<File> mGet(Message<?> message, Session<F> session, String remoteD
return files;
}

private List<File> mGetWithRecursion(Message<?> message, Session<F> session, String remoteDirectory,
String remoteFilename) throws IOException {
List<File> files = new ArrayList<File>();
@SuppressWarnings("unchecked")
List<AbstractFileInfo<F>> fileNames = (List<AbstractFileInfo<F>>) this.ls(session, remoteDirectory);
if (fileNames.size() == 0 && this.options.contains(Option.EXCEPTION_WHEN_EMPTY)) {
throw new MessagingException("No files found at " + remoteDirectory
+ " with pattern " + remoteFilename);
}
for (AbstractFileInfo<F> lsEntry : fileNames) {
String fullFileName = remoteDirectory + this.getFilename(lsEntry);
/*
* With recursion, the filename might contain subdirectory information
* normalize each file separately.
*/
String fileName = this.getRemoteFilename(fullFileName);
String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName);
File file = this.get(message, session, actualRemoteDirectory,
fullFileName, fileName, false);
files.add(file);
}
return files;
}
Copy link
Member

@artembilan artembilan Oct 17, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't see 'Recursion'. Or this.getFilename(lsEntry) really return full relative path from remoteDirectory ?

Copy link
Contributor Author

@garyrussell garyrussell Oct 17, 2013

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The recursion is done in ls() and, in that case, each returned filename is enhanced to include the relative path (relative to the requested directory).


private String getRemoteDirectory(String remoteFilePath, String remoteFilename) {
String remoteDir = remoteFilePath.substring(0, remoteFilePath.lastIndexOf(remoteFilename));
if (remoteDir.length() == 0) {
Expand Down Expand Up @@ -626,8 +700,11 @@ private String generateLocalFileName(Message<?> message, String remoteFileName){

abstract protected String getFilename(F file);

abstract protected String getFilename(AbstractFileInfo<F> file);

abstract protected long getModified(F file);

abstract protected List<AbstractFileInfo<F>> asFileInfoList(Collection<F> files);

abstract protected F enhanceNameWithSubDirectory(F file, String directory);
}
116 changes: 95 additions & 21 deletions ...a/org/springframework/integration/file/remote/gateway/RemoteFileOutboundGatewayTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,21 +86,6 @@ public void testBadFilterGet() throws Exception {
}
}

@Test
public void testBadFilterMGet() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "mget", "payload");
gw.setFilter(new TestPatternFilter(""));
try {
gw.onInit();
fail("Exception expected");
}
catch (IllegalArgumentException e) {
assertTrue(e.getMessage().startsWith("Filters are not supported"));
}
}

@Test
public void testBadFilterRm() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Expand Down Expand Up @@ -389,9 +374,6 @@ public Object answer(InvocationOnMock invocation) throws Throwable {
assertEquals("foo/bar", madeDirs.get(1));
}

/**
* @return
*/
public TestLsEntry[] fileList() {
TestLsEntry[] files = new TestLsEntry[6];
files[0] = new TestLsEntry("f2", 123, false, false, 1234, "-r--r--r--");
Expand Down Expand Up @@ -424,6 +406,83 @@ public void testLs_f() throws Exception {
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}

public TestLsEntry[] level1List() {
return new TestLsEntry[] {
new TestLsEntry("f1", 123, false, false, 1234, "-r--r--r--"),
new TestLsEntry("d1", 0, true, false, 12345, "drw-r--r--"),
new TestLsEntry("f2", 12345, false, false, 123456, "-rw-r--r--")
};
}

public TestLsEntry[] level2List() {
return new TestLsEntry[] {
new TestLsEntry("d2", 0, true, false, 12345, "drw-r--r--"),
new TestLsEntry("f3", 12345, false, false, 123456, "-rw-r--r--")
};
}

public TestLsEntry[] level3List() {
return new TestLsEntry[] {
new TestLsEntry("f4", 12345, false, false, 123456, "-rw-r--r--")
};
}

@Test
public void testLs_f_R() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Session session = mock(Session.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "ls", "payload");
gw.setOptions("-f -R");
gw.afterPropertiesSet();
when(sessionFactory.getSession()).thenReturn(session);
TestLsEntry[] level1 = level1List();
TestLsEntry[] level2 = level2List();
TestLsEntry[] level3 = level3List();
when(session.list("testremote/x/")).thenReturn(level1);
when(session.list("testremote/x/d1/")).thenReturn(level2);
when(session.list("testremote/x/d1/d2/")).thenReturn(level3);
@SuppressWarnings("unchecked")
Message<List<TestLsEntry>> out = (Message<List<TestLsEntry>>) gw
.handleRequestMessage(new GenericMessage<String>("testremote/x"));
assertEquals(4, out.getPayload().size());
assertEquals("f1", out.getPayload().get(0).getFilename());
assertEquals("d1/d2/f4", out.getPayload().get(1).getFilename());
assertEquals("d1/f3", out.getPayload().get(2).getFilename());
assertEquals("f2", out.getPayload().get(3).getFilename());
assertEquals("testremote/x/",
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}

@Test
public void testLs_f_R_dirs() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Session session = mock(Session.class);
TestRemoteFileOutboundGateway gw = new TestRemoteFileOutboundGateway
(sessionFactory, "ls", "payload");
gw.setOptions("-f -R -dirs");
gw.afterPropertiesSet();
when(sessionFactory.getSession()).thenReturn(session);
TestLsEntry[] level1 = level1List();
TestLsEntry[] level2 = level2List();
TestLsEntry[] level3 = level3List();
when(session.list("testremote/x/")).thenReturn(level1);
when(session.list("testremote/x/d1/")).thenReturn(level2);
when(session.list("testremote/x/d1/d2/")).thenReturn(level3);
@SuppressWarnings("unchecked")
Message<List<TestLsEntry>> out = (Message<List<TestLsEntry>>) gw
.handleRequestMessage(new GenericMessage<String>("testremote/x"));
assertEquals(6, out.getPayload().size());
assertEquals("f1", out.getPayload().get(0).getFilename());
assertEquals("d1", out.getPayload().get(1).getFilename());
assertEquals("d1/d2", out.getPayload().get(2).getFilename());
assertEquals("d1/d2/f4", out.getPayload().get(3).getFilename());
assertEquals("d1/f3", out.getPayload().get(4).getFilename());
assertEquals("f2", out.getPayload().get(5).getFilename());
assertEquals("testremote/x/",
out.getHeaders().get(FileHeaders.REMOTE_DIRECTORY));
}

@Test
public void testLs_None() throws Exception {
SessionFactory sessionFactory = mock(SessionFactory.class);
Expand Down Expand Up @@ -775,6 +834,11 @@ protected String getFilename(TestLsEntry file) {
return file.getFilename();
}

@Override
protected String getFilename(AbstractFileInfo<TestLsEntry> file) {
return file.getFilename();
}

@Override
protected long getModified(TestLsEntry file) {
return file.getModified();
Expand All @@ -786,18 +850,24 @@ protected List<AbstractFileInfo<TestLsEntry>> asFileInfoList(
return new ArrayList<AbstractFileInfo<TestLsEntry>>(files);
}

@Override
protected TestLsEntry enhanceNameWithSubDirectory(TestLsEntry file, String directory) {
file.setFilename(directory + file.getFilename());
return file;
}

}

class TestLsEntry extends AbstractFileInfo<TestLsEntry> {

private final String filename;
private final int size;
private volatile String filename;
private final long size;
private final boolean dir;
private final boolean link;
private final long modified;
private final String permissions;

public TestLsEntry(String filename, int size, boolean dir, boolean link,
public TestLsEntry(String filename, long size, boolean dir, boolean link,
long modified, String permissions) {
this.filename = filename;
this.size = size;
Expand Down Expand Up @@ -835,6 +905,10 @@ public TestLsEntry getFileInfo() {
return this;
}

public void setFilename(String filename) {
this.filename = filename;
}

}

class TestPatternFilter extends AbstractSimplePatternFileListFilter<TestLsEntry>{
Expand Down
Loading