INT-4115: (S)FtpPersistentFileFilter by Default

JIRA: https://jira.spring.io/browse/INT-4115 Apply `(S)FtpPersistentAcceptOnceFileListFilter` for the `(S)FtpInboundFileSynchronizer` by default to avoid cases to sync the same remote files to the local directory again. Especially when `localFileName` strategy is applied and we end up with new local files, but with the same remote content Accept `(S)FtpPersistentAcceptOnceFileListFilter` for streaming adapters Make `doSetFilter()` as `protected final` Fix tests after rebase Fix "What's New" after rebase Compose `PersistentAcceptOnceFileListFilter` together with the regex or pattern filters Document such a behavior Address PR comments for formatting and typo
spring-projects · Mar 14, 2017 · 10ce68d · 10ce68d
1 parent d3fb8b8
commit 10ce68d
Show file tree

Hide file tree

Showing 29 changed files with 328 additions and 112 deletions.
diff --git a/...pringframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java b/...pringframework/integration/file/config/AbstractRemoteFileInboundChannelAdapterParser.java
@@ -25,6 +25,7 @@
 import org.springframework.integration.config.ExpressionFactoryBean;
 import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
 import org.springframework.util.StringUtils;
@@ -36,6 +37,7 @@
  * @author Mark Fisher
  * @author Gary Russell
  * @author Artem Bilan
+ *
  * @since 2.0
  */
 public abstract class AbstractRemoteFileInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser {
@@ -59,8 +61,10 @@ protected final BeanMetadataElement parseSource(Element element, ParserContext p
 		String remoteFileSeparator = element.getAttribute("remote-file-separator");
 		synchronizerBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
 		IntegrationNamespaceUtils.setValueIfAttributeDefined(synchronizerBuilder, element, "temporary-file-suffix");
+
 		FileParserUtils.configureFilter(synchronizerBuilder, element, parserContext,
-				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass());
+				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass(),
+				getPersistentAcceptOnceFileListFilterClass());
 
 		// build the MessageSource
 		BeanDefinitionBuilder messageSourceBuilder =
@@ -94,4 +98,6 @@ protected final BeanMetadataElement parseSource(Element element, ParserContext p
 
 	protected abstract Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass();
 
+	protected abstract Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass();
+
 }
diff --git a/...ework/integration/file/config/AbstractRemoteFileStreamingInboundChannelAdapterParser.java b/...ework/integration/file/config/AbstractRemoteFileStreamingInboundChannelAdapterParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 the original author or authors.
+ * Copyright 2016-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@
 import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
 import org.springframework.integration.core.MessageSource;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.RemoteFileOperations;
 import org.springframework.util.StringUtils;
@@ -33,6 +34,8 @@
  * Abstract base class for parsing remote file streaming inbound channel adapters.
  *
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 4.3
  */
 public abstract class AbstractRemoteFileStreamingInboundChannelAdapterParser
@@ -56,7 +59,7 @@ protected final BeanMetadataElement parseSource(Element element, ParserContext p
 		String remoteFileSeparator = element.getAttribute("remote-file-separator");
 		messageSourceBuilder.addPropertyValue("remoteFileSeparator", remoteFileSeparator);
 		FileParserUtils.configureFilter(messageSourceBuilder, element, parserContext,
-				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass());
+				getSimplePatternFileListFilterClass(), getRegexPatternFileListFilterClass(), getPersistentAcceptOnceFileListFilterClass());
 
 		String comparator = element.getAttribute("comparator");
 		if (StringUtils.hasText(comparator)) {
@@ -74,4 +77,6 @@ protected final BeanMetadataElement parseSource(Element element, ParserContext p
 
 	protected abstract Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass();
 
+	protected abstract Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass();
+
 }
diff --git a/...ation-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java b/...ation-file/src/main/java/org/springframework/integration/file/config/FileParserUtils.java
@@ -20,12 +20,16 @@
 
 import org.springframework.beans.factory.config.BeanDefinition;
 import org.springframework.beans.factory.support.BeanDefinitionBuilder;
+import org.springframework.beans.factory.support.ManagedList;
 import org.springframework.beans.factory.xml.ParserContext;
 import org.springframework.integration.config.xml.IntegrationNamespaceUtils;
 import org.springframework.integration.file.DefaultFileNameGenerator;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
+import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.RemoteFileOperations;
+import org.springframework.integration.metadata.SimpleMetadataStore;
 import org.springframework.util.StringUtils;
 
 /**
@@ -34,6 +38,7 @@
  * @author David Turanski
  * @author Gary Russell
  * @author Artem Bilan
+ *
  * @since 3.0
  *
  */
@@ -93,7 +98,8 @@ public static BeanDefinition parseRemoteFileTemplate(Element element, ParserCont
 	}
 
 	static void configureFilter(BeanDefinitionBuilder synchronizerBuilder, Element element, ParserContext parserContext,
-			Class<? extends FileListFilter<?>> patternClass, Class<? extends FileListFilter<?>> regexClass) {
+			Class<? extends FileListFilter<?>> patternClass, Class<? extends FileListFilter<?>> regexClass,
+			Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> persistentAcceptOnceFileListFilterClass) {
 		String filter = element.getAttribute("filter");
 		String filterExpression = element.getAttribute("filter-expression");
 		String fileNamePattern = element.getAttribute("filename-pattern");
@@ -132,16 +138,44 @@ else if (hasFilterExpression) {
 				synchronizerBuilder.addPropertyValue("filter", expressionFilterBeanDefinition);
 			}
 			else if (hasFileNamePattern) {
-				BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition(patternClass);
-				filterBuilder.addConstructorArgValue(fileNamePattern);
-				synchronizerBuilder.addPropertyValue("filter", filterBuilder.getBeanDefinition());
+				BeanDefinition patternFilter =
+						BeanDefinitionBuilder.genericBeanDefinition(patternClass)
+								.addConstructorArgValue(fileNamePattern)
+								.getBeanDefinition();
+
+				composeFilters(synchronizerBuilder, persistentAcceptOnceFileListFilterClass, patternFilter);
 			}
-			else if (hasFileNameRegex) {
-				BeanDefinitionBuilder filterBuilder = BeanDefinitionBuilder.genericBeanDefinition(regexClass);
-				filterBuilder.addConstructorArgValue(fileNameRegex);
-				synchronizerBuilder.addPropertyValue("filter", filterBuilder.getBeanDefinition());
+			else {
+				BeanDefinition regexFilter = BeanDefinitionBuilder.genericBeanDefinition(regexClass)
+						.addConstructorArgValue(fileNameRegex)
+						.getBeanDefinition();
+
+				composeFilters(synchronizerBuilder, persistentAcceptOnceFileListFilterClass, regexFilter);
 			}
 		}
 	}
 
+	private static void composeFilters(BeanDefinitionBuilder synchronizerBuilder,
+			Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> persistentAcceptOnceFileListFilterClass,
+			BeanDefinition filter) {
+		BeanDefinition persistentFilter =
+				BeanDefinitionBuilder.genericBeanDefinition(persistentAcceptOnceFileListFilterClass)
+						.addConstructorArgValue(
+								BeanDefinitionBuilder
+										.genericBeanDefinition(SimpleMetadataStore.class)
+										.getBeanDefinition())
+						.addConstructorArgValue("remoteFileMessageSource")
+						.getBeanDefinition();
+
+		ManagedList<BeanDefinition> filters = new ManagedList<>();
+		filters.add(filter);
+		filters.add(persistentFilter);
+
+		BeanDefinition compositeFilterDefinition =
+				BeanDefinitionBuilder.genericBeanDefinition(CompositeFileListFilter.class)
+						.addConstructorArgValue(filters)
+						.getBeanDefinition();
+		synchronizerBuilder.addPropertyValue("filter", compositeFilterDefinition);
+	}
+
 }
diff --git a/...in/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java b/...in/java/org/springframework/integration/file/dsl/RemoteFileInboundChannelAdapterSpec.java
@@ -27,7 +27,6 @@
 import org.springframework.integration.dsl.MessageSourceSpec;
 import org.springframework.integration.expression.FunctionExpression;
 import org.springframework.integration.file.FileReadingMessageSource;
-import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer;
@@ -51,8 +50,6 @@ public abstract class RemoteFileInboundChannelAdapterSpec<F, S extends RemoteFil
 
 	protected final AbstractInboundFileSynchronizer<F> synchronizer;
 
-	private CompositeFileListFilter<F> filter;
-
 	private ExpressionFileListFilter<F> expressionFileListFilter;
 
 	protected RemoteFileInboundChannelAdapterSpec(AbstractInboundFileSynchronizer<F> synchronizer) {
@@ -172,19 +169,7 @@ public S remoteDirectoryExpression(Expression remoteDirectoryExpression) {
 	 * @return the spec.
 	 */
 	public S filter(FileListFilter<F> filter) {
-		if (this.filter == null) {
-			if (filter instanceof CompositeFileListFilter) {
-				this.filter = (CompositeFileListFilter<F>) filter;
-			}
-			else {
-				this.filter = new CompositeFileListFilter<>();
-				this.filter.addFilter(filter);
-			}
-			this.synchronizer.setFilter(this.filter);
-		}
-		else {
-			this.filter.addFilter(filter);
-		}
+		this.synchronizer.setFilter(filter);
 		return _this();
 	}
 

diff --git a/...rg/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java b/...rg/springframework/integration/file/dsl/RemoteFileStreamingInboundChannelAdapterSpec.java
@@ -25,7 +25,6 @@
 import org.springframework.integration.dsl.MessageSourceSpec;
 import org.springframework.integration.expression.FunctionExpression;
 import org.springframework.integration.file.FileReadingMessageSource;
-import org.springframework.integration.file.filters.CompositeFileListFilter;
 import org.springframework.integration.file.filters.ExpressionFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.AbstractRemoteFileStreamingMessageSource;
@@ -49,8 +48,6 @@ public abstract class RemoteFileStreamingInboundChannelAdapterSpec<F,
 		extends MessageSourceSpec<S, MS>
 		implements ComponentsRegistration {
 
-	private CompositeFileListFilter<F> filter;
-
 	private ExpressionFileListFilter<F> expressionFileListFilter;
 
 	/**
@@ -101,19 +98,7 @@ public S remoteDirectory(Function<Message<?>, String> remoteDirectoryFunction) {
 	 * @return the spec.
 	 */
 	public S filter(FileListFilter<F> filter) {
-		if (this.filter == null) {
-			if (filter instanceof CompositeFileListFilter) {
-				this.filter = (CompositeFileListFilter<F>) filter;
-			}
-			else {
-				this.filter = new CompositeFileListFilter<F>();
-				this.filter.addFilter(filter);
-			}
-			this.target.setFilter(this.filter);
-		}
-		else {
-			this.filter.addFilter(filter);
-		}
+		this.target.setFilter(filter);
 		return _this();
 	}
 

diff --git a/...org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java b/...org/springframework/integration/file/remote/AbstractRemoteFileStreamingMessageSource.java
@@ -46,6 +46,8 @@
  * referencing a remote file.
  *
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 4.3
  *
  */
@@ -111,6 +113,10 @@ public void setRemoteFileSeparator(String remoteFileSeparator) {
 	 * @param filter the file list filter.
 	 */
 	public void setFilter(FileListFilter<F> filter) {
+		doSetFilter(filter);
+	}
+
+	protected final void doSetFilter(FileListFilter<F> filter) {
 		this.filter = filter;
 	}
 

diff --git a/...springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java b/...springframework/integration/file/remote/synchronizer/AbstractInboundFileSynchronizer.java
@@ -152,11 +152,14 @@ public void setRemoteDirectory(String remoteDirectory) {
 
 	/**
 	 * Specify an expression that evaluates to the full path to the remote directory.
-	 *
 	 * @param remoteDirectoryExpression The remote directory expression.
 	 * @since 4.2
 	 */
 	public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) {
+		doSetRemoteDirectoryExpression(remoteDirectoryExpression);
+	}
+
+	protected final void doSetRemoteDirectoryExpression(Expression remoteDirectoryExpression) {
 		Assert.notNull(remoteDirectoryExpression, "'remoteDirectoryExpression' must not be null");
 		this.remoteDirectoryExpression = remoteDirectoryExpression;
 	}
@@ -166,6 +169,10 @@ public void setRemoteDirectoryExpression(Expression remoteDirectoryExpression) {
 	 * @param filter the file list filter.
 	 */
 	public void setFilter(FileListFilter<F> filter) {
+		doSetFilter(filter);
+	}
+
+	protected final void doSetFilter(FileListFilter<F> filter) {
 		this.filter = filter;
 	}
 

diff --git a/.../main/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParser.java b/.../main/java/org/springframework/integration/ftp/config/FtpInboundChannelAdapterParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2012 the original author or authors.
+ * Copyright 2002-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,10 @@
 package org.springframework.integration.ftp.config;
 
 import org.springframework.integration.file.config.AbstractRemoteFileInboundChannelAdapterParser;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.synchronizer.InboundFileSynchronizer;
+import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
 import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
 import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizer;
@@ -29,6 +31,8 @@
  *
  * @author Mark Fisher
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 2.0
  */
 public class FtpInboundChannelAdapterParser extends AbstractRemoteFileInboundChannelAdapterParser {
@@ -53,4 +57,9 @@ protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass(
 		return FtpRegexPatternFileListFilter.class;
 	}
 
+	@Override
+	protected Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass() {
+		return FtpPersistentAcceptOnceFileListFilter.class;
+	}
+
 }
diff --git a/...a/org/springframework/integration/ftp/config/FtpStreamingInboundChannelAdapterParser.java b/...a/org/springframework/integration/ftp/config/FtpStreamingInboundChannelAdapterParser.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 the original author or authors.
+ * Copyright 2016-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,15 +18,19 @@
 
 import org.springframework.integration.core.MessageSource;
 import org.springframework.integration.file.config.AbstractRemoteFileStreamingInboundChannelAdapterParser;
+import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.file.filters.FileListFilter;
 import org.springframework.integration.file.remote.RemoteFileOperations;
+import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter;
 import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter;
 import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter;
 import org.springframework.integration.ftp.inbound.FtpStreamingMessageSource;
 import org.springframework.integration.ftp.session.FtpRemoteFileTemplate;
 
 /**
  * @author Gary Russell
+ * @author Artem Bilan
+ *
  * @since 4.3
  *
  */
@@ -52,4 +56,9 @@ protected Class<? extends FileListFilter<?>> getRegexPatternFileListFilterClass(
 		return FtpRegexPatternFileListFilter.class;
 	}
 
+	@Override
+	protected Class<? extends AbstractPersistentAcceptOnceFileListFilter<?>> getPersistentAcceptOnceFileListFilterClass() {
+		return FtpPersistentAcceptOnceFileListFilter.class;
+	}
+
 }