INT-4231: Add FileExistsMode.REPLACE_IF_MODIFIED

spring-integration-file/src/main/java/org/springframework/integration/file/FileWritingMessageHandler.java

@@ -213,8 +213,18 @@ public void setTemporaryFileSuffix(String temporaryFileSuffix) {
 	 * <p>
 	 * Otherwise the LockRegistry is set to {@link PassThruLockRegistry} which
 	 * has no effect.
+	 * <p>
+	 * With {@link FileExistsMode#REPLACE_IF_MODIFIED}, if the file exists,
+	 * it is only replaced if its last modified timestamp is different to the
+	 * source; otherwise, the write is ignored. For {@link File} payloads,
+	 * the actual timestamp of the {@link File} is compared; for other payloads,
+	 * the {@link FileHeaders#SET_MODIFIED} is compared to the existing file.
+	 * If the header is missing, or its value is not a {@link Number}, the file
+	 * is always replaced. This mode will typically only make sense if
+	 * {@link #setPreserveTimestamp(boolean) preserveTimestamp} is true.
 	 *
 	 * @param fileExistsMode Must not be null
+	 * @see #setPreserveTimestamp(boolean)
 	 */
 	public void setFileExistsMode(FileExistsMode fileExistsMode) {
 
@@ -426,26 +436,30 @@ protected Object handleRequestMessage(Message<?> requestMessage) {
 		File tempFile = new File(destinationDirectoryToUse, generatedFileName + this.temporaryFileSuffix);
 		File resultFile = new File(destinationDirectoryToUse, generatedFileName);
 
-		if (FileExistsMode.FAIL.equals(this.fileExistsMode) && resultFile.exists()) {
+		boolean exists = resultFile.exists();
+		if (exists && FileExistsMode.FAIL.equals(this.fileExistsMode)) {
 			throw new MessageHandlingException(requestMessage,
 					"The destination file already exists at '" + resultFile.getAbsolutePath() + "'.");
 		}
 
-		final boolean ignore = FileExistsMode.IGNORE.equals(this.fileExistsMode) &&
-				(resultFile.exists() ||
-						(StringUtils.hasText(this.temporaryFileSuffix) && tempFile.exists()));
-
+		Object timestamp = requestMessage.getHeaders().get(FileHeaders.SET_MODIFIED);
+		if (payload instanceof File) {
+			timestamp = ((File) payload).lastModified();
+		}
+		boolean ignore = (FileExistsMode.IGNORE.equals(this.fileExistsMode)
+						&& (exists || (StringUtils.hasText(this.temporaryFileSuffix) && tempFile.exists())))
+				|| ((exists && FileExistsMode.REPLACE_IF_MODIFIED.equals(this.fileExistsMode))
+						&& (timestamp instanceof Number
+								&& ((Number) timestamp).longValue() == resultFile.lastModified()));
 		if (!ignore) {
 			try {
-				Object timestamp = requestMessage.getHeaders().get(FileHeaders.SET_MODIFIED);
-				if (!resultFile.exists() &&
+				if (!exists &&
 						generatedFileName.replaceAll("/", Matcher.quoteReplacement(File.separator))
 								.contains(File.separator)) {
 					resultFile.getParentFile().mkdirs(); //NOSONAR - will fail on the writing below
 				}
 				if (payload instanceof File) {
 					resultFile = handleFileMessage((File) payload, tempFile, resultFile);
-					timestamp = ((File) payload).lastModified();
 				}
 				else if (payload instanceof InputStream) {
 					resultFile = handleInputStreamMessage((InputStream) payload, originalFileFromHeader, tempFile,
@@ -711,6 +725,7 @@ private File determineFileToWrite(File resultFile, File tempFile) {
 			case FAIL:
 			case IGNORE:
 			case REPLACE:
+			case REPLACE_IF_MODIFIED:
 				fileToWriteTo = tempFile;
 				break;
 			default:

spring-integration-file/src/main/java/org/springframework/integration/file/remote/RemoteFileTemplate.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2016 the original author or authors.
+ * Copyright 2013-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -282,6 +282,8 @@ private String send(final Message<?> message, final String subDirectory, final F
 		Assert.notNull(this.directoryExpressionProcessor, "'remoteDirectoryExpression' is required");
 		Assert.isTrue(!FileExistsMode.APPEND.equals(mode) || !this.useTemporaryFileName,
 				"Cannot append when using a temporary file name");
+		Assert.isTrue(!FileExistsMode.REPLACE_IF_MODIFIED.equals(mode),
+				"FilExistsMode.REPLACE_IF_MODIFIED can only be used for local files");
 		final StreamHolder inputStreamHolder = this.payloadToInputStream(message);
 		if (inputStreamHolder != null) {
 			try {

spring-integration-file/src/main/java/org/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java

@@ -587,7 +587,7 @@ private Object doGet(final Message<?> requestMessage) {
 		}
 		else {
 			payload = this.remoteFileTemplate.execute(session1 ->
-					get(requestMessage, session1, remoteDir, remoteFilePath, remoteFilename, true));
+					get(requestMessage, session1, remoteDir, remoteFilePath, remoteFilename, null));
 		}
 		return getMessageBuilderFactory().withPayload(payload)
 				.setHeader(FileHeaders.REMOTE_DIRECTORY, remoteDir)
@@ -831,33 +831,38 @@ protected void purgeDots(List<F> lsFiles) {
 	 * Copy a remote file to the configured local directory.
 	 *
 	 *
-	 * @param message The message.
-	 * @param session The session.
-	 * @param remoteDir The remote directory.
-	 * @param remoteFilePath The remote file path.
-	 * @param remoteFilename The remote file name.
-	 * @param lsFirst true to execute an 'ls' command first.
+	 * @param message the message.
+	 * @param session the session.
+	 * @param remoteDir the remote directory.
+	 * @param remoteFilePath the remote file path.
+	 * @param remoteFilename the remote file name.
+	 * @param fileInfoParam the remote file info; if null we will execute an 'ls' command
+	 * first.
 	 * @return The file.
 	 * @throws IOException Any IOException.
 	 */
 	protected File get(Message<?> message, Session<F> session, String remoteDir, String remoteFilePath,
-			String remoteFilename, boolean lsFirst) throws IOException {
-		F[] files = null;
-		if (lsFirst) {
-			files = session.list(remoteFilePath);
+			String remoteFilename, F fileInfoParam) throws IOException {
+		F fileInfo = fileInfoParam;
+		if (fileInfo == null) {
+			F[] files = session.list(remoteFilePath);
 			if (files == null) {
 				throw new MessagingException("Session returned null when listing " + remoteFilePath);
 			}
-			if (files.length != 1 || isDirectory(files[0]) || isLink(files[0])) {
+			if (files.length != 1 || files[0] == null || isDirectory(files[0]) || isLink(files[0])) {
 				throw new MessagingException(remoteFilePath + " is not a file");
 			}
+			fileInfo = files[0];
 		}
 		File localFile =
 				new File(generateLocalDirectory(message, remoteDir), generateLocalFileName(message, remoteFilename));
 		FileExistsMode fileExistsMode = this.fileExistsMode;
 		boolean appending = FileExistsMode.APPEND.equals(fileExistsMode);
-		boolean replacing = FileExistsMode.REPLACE.equals(fileExistsMode);
-		if (!localFile.exists() || appending || replacing) {
+		boolean exists = localFile.exists();
+		boolean replacing = FileExistsMode.REPLACE.equals(fileExistsMode)
+				|| (exists && FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)
+						&& localFile.lastModified() != getModified(fileInfo));
+		if (!exists || appending || replacing) {
 			OutputStream outputStream;
 			String tempFileName = localFile.getAbsolutePath() + this.remoteFileTemplate.getTemporaryFileSuffix();
 			File tempFile = new File(tempFileName);
@@ -898,11 +903,14 @@ protected File get(Message<?> message, Session<F> session, String remoteDir, Str
 			if (!appending && !tempFile.renameTo(localFile)) {
 				throw new MessagingException("Failed to rename local file");
 			}
-			if (lsFirst && this.options.contains(Option.PRESERVE_TIMESTAMP)) {
-				localFile.setLastModified(getModified(files[0]));
+			if (this.options.contains(Option.PRESERVE_TIMESTAMP)) {
+				localFile.setLastModified(getModified(fileInfo));
 			}
 		}
-		else if (FileExistsMode.IGNORE != fileExistsMode) {
+		else if (FileExistsMode.REPLACE_IF_MODIFIED.equals(fileExistsMode)) {
+			logger.debug("Local file '" + localFile + "' has the same modified timestamp, ignored");
+		}
+		else if (!FileExistsMode.IGNORE.equals(fileExistsMode)) {
 			throw new MessageHandlingException(message, "Local file " + localFile + " already exists");
 		}
 		else {
@@ -955,10 +963,7 @@ private List<File> mGetWithoutRecursion(Message<?> message, Session<F> session,
 				String fileName = this.getRemoteFilename(fullFileName);
 				String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName);
 				File file = get(message, session, actualRemoteDirectory,
-						fullFileName, fileName, false);
-				if (this.options.contains(Option.PRESERVE_TIMESTAMP)) {
-					file.setLastModified(getModified(lsEntry.getFileInfo()));
-				}
+						fullFileName, fileName, lsEntry.getFileInfo());
 				files.add(file);
 			}
 		}
@@ -1001,10 +1006,7 @@ private List<File> mGetWithRecursion(Message<?> message, Session<F> session, Str
 				String fileName = this.getRemoteFilename(fullFileName);
 				String actualRemoteDirectory = this.getRemoteDirectory(fullFileName, fileName);
 				File file = get(message, session, actualRemoteDirectory,
-						fullFileName, fileName, false);
-				if (this.options.contains(Option.PRESERVE_TIMESTAMP)) {
-					file.setLastModified(getModified(lsEntry.getFileInfo()));
-				}
+						fullFileName, fileName, lsEntry.getFileInfo());
 				files.add(file);
 			}
 		}

spring-integration-file/src/main/java/org/springframework/integration/file/support/FileExistsMode.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2002-2016 the original author or authors.
+ * Copyright 2002-2017 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -55,7 +55,14 @@ public enum FileExistsMode {
 	/**
 	 * If the file already exists, replace it.
 	 */
-	REPLACE;
+	REPLACE,
+
+	/**
+	 * If the file already exists, replace it only if the last modified time
+	 * is different. Only applies to local files.
+	 * @since 5.0
+	 */
+	REPLACE_IF_MODIFIED;
 
 	/**
 	 * For a given non-null and not-empty input string, this method returns the

spring-integration-file/src/main/resources/org/springframework/integration/file/config/spring-integration-file-5.0.xsd

@@ -828,6 +828,15 @@ Only files matching this regular expression will be picked up by this adapter.
 					]]></xsd:documentation>
 				</xsd:annotation>
 			</xsd:enumeration>
+			<xsd:enumeration value="REPLACE_IF_MODIFIED">
+				<xsd:annotation>
+					<xsd:documentation><![CDATA[
+						If the local file already exists, it will be overwritten only
+						if the last modified timestamp does not match the source
+						time stamp. Only applies to local files.
+					]]></xsd:documentation>
+				</xsd:annotation>
+			</xsd:enumeration>
 			<xsd:enumeration value="APPEND">
 				<xsd:annotation>
 					<xsd:documentation><![CDATA[
@@ -1004,6 +1013,12 @@ Only files matching this regular expression will be picked up by this adapter.
 
 					This is the default behavior when writing files. If the
 					target file already exists, it will be overwritten.
+
+					REPLACE_IF_MODIFIED:
+
+					If the local file already exists, it will be overwritten only
+					if the last modified timestamp does not match the source
+					timestamp. Only applies to local files.
 				]]></xsd:documentation>
 			</xsd:annotation>
 			<xsd:simpleType>

spring-integration-file/src/test/java/org/springframework/integration/file/FileWritingMessageHandlerTests.java

@@ -518,6 +518,61 @@ public void noFlushAppend() throws Exception {
 		handler.stop();
 	}
 
+	@Test
+	public void replaceIfDifferent() throws IOException {
+		QueueChannel output = new QueueChannel();
+		this.handler.setOutputChannel(output);
+		this.handler.setPreserveTimestamp(true);
+		this.handler.setFileExistsMode(FileExistsMode.REPLACE_IF_MODIFIED);
+		this.handler.handleMessage(MessageBuilder.withPayload("foo")
+				.setHeader(FileHeaders.FILENAME, "replaceIfDifferent.txt")
+				.setHeader(FileHeaders.SET_MODIFIED, 42_000_000)
+				.build());
+		Message<?> result = output.receive(0);
+		assertFileContentIs(result, "foo");
+		assertLastModifiedIs(result, 42_000_000);
+		this.handler.handleMessage(MessageBuilder.withPayload("bar")
+				.setHeader(FileHeaders.FILENAME, "replaceIfDifferent.txt")
+				.setHeader(FileHeaders.SET_MODIFIED, 42_000_000)
+				.build());
+		result = output.receive(0);
+		assertFileContentIs(result, "foo"); // no overwrite - timestamp same
+		assertLastModifiedIs(result, 42_000_000);
+		this.handler.handleMessage(MessageBuilder.withPayload("bar")
+				.setHeader(FileHeaders.FILENAME, "replaceIfDifferent.txt")
+				.setHeader(FileHeaders.SET_MODIFIED, 43_000_000)
+				.build());
+		result = output.receive(0);
+		assertFileContentIs(result, "bar");
+		assertLastModifiedIs(result, 43_000_000);
+	}
+
+	@Test
+	public void replaceIfDifferentFile() throws IOException {
+		File file = new File(this.temp.newFolder(), "foo.txt");
+		FileCopyUtils.copy("foo".getBytes(), new FileOutputStream(file));
+		file.setLastModified(42_000_000);
+		QueueChannel output = new QueueChannel();
+		this.handler.setOutputChannel(output);
+		this.handler.setPreserveTimestamp(true);
+		this.handler.setFileExistsMode(FileExistsMode.REPLACE_IF_MODIFIED);
+		this.handler.handleMessage(MessageBuilder.withPayload(file).build());
+		Message<?> result = output.receive(0);
+		assertFileContentIs(result, "foo");
+		assertLastModifiedIs(result, 42_000_000);
+		FileCopyUtils.copy("bar".getBytes(), new FileOutputStream(file));
+		file.setLastModified(42_000_000);
+		this.handler.handleMessage(MessageBuilder.withPayload(file).build());
+		result = output.receive(0);
+		assertFileContentIs(result, "foo"); // no overwrite - timestamp same
+		assertLastModifiedIs(result, 42_000_000);
+		file.setLastModified(43_000_000);
+		this.handler.handleMessage(MessageBuilder.withPayload(file).build());
+		result = output.receive(0);
+		assertFileContentIs(result, "bar");
+		assertLastModifiedIs(result, 43_000_000);
+	}
+
 	void assertFileContentIsMatching(Message<?> result) throws IOException {
 		assertFileContentIs(result, SAMPLE_CONTENT);
 	}

spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests-context.xml

@@ -53,6 +53,7 @@
 							  command="mget"
 							  expression="payload"
 							  command-options="-R -P"
+							  mode="REPLACE_IF_MODIFIED"
 							  filter="starDotTxtFilter"
 							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('ftpSource', 'localTarget')"

spring-integration-ftp/src/test/java/org/springframework/integration/ftp/outbound/FtpServerOutboundTests.java

@@ -36,6 +36,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
 
+import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -48,6 +49,7 @@
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.net.ftp.FTPClient;
 import org.apache.commons.net.ftp.FTPFile;
 import org.hamcrest.Matchers;
@@ -247,9 +249,11 @@ public void testMGETOnNullDir() throws IOException {
 
 	@Test
 	@SuppressWarnings("unchecked")
-	public void testInt3172LocalDirectoryExpressionMGETRecursive() {
+	public void testInt3172LocalDirectoryExpressionMGETRecursive() throws IOException {
 		String dir = "ftpSource/";
 		long modified = setModifiedOnSource1();
+		File secondRemote = new File(getSourceRemoteDirectory(), "ftpSource2.txt");
+		secondRemote.setLastModified(System.currentTimeMillis() - 1_000_000);
 		this.inboundMGetRecursive.send(new GenericMessage<Object>("*"));
 		Message<?> result = this.output.receive(1000);
 		assertNotNull(result);
@@ -268,6 +272,30 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() {
 		assertThat(localFiles.get(2).getPath().replaceAll(quoteReplacement(File.separator), "/"),
 				containsString(dir + "subFtpSource"));
 
+		File secondTarget = new File(getTargetLocalDirectory() + File.separator + "ftpSource", "localTarget2.txt");
+		ByteArrayOutputStream remoteContents = new ByteArrayOutputStream();
+		ByteArrayOutputStream localContents = new ByteArrayOutputStream();
+		FileUtils.copyFile(secondRemote, remoteContents);
+		FileUtils.copyFile(secondTarget, localContents);
+		String localAsString = new String(localContents.toByteArray());
+		assertEquals(new String(remoteContents.toByteArray()), localAsString);
+		long oldLastModified = secondRemote.lastModified();
+		FileUtils.copyInputStreamToFile(new ByteArrayInputStream("junk".getBytes()), secondRemote);
+		long newLastModified = secondRemote.lastModified();
+		secondRemote.setLastModified(oldLastModified);
+		this.inboundMGetRecursive.send(new GenericMessage<Object>("*"));
+		this.output.receive(0);
+		localContents = new ByteArrayOutputStream();
+		FileUtils.copyFile(secondTarget, localContents);
+		assertEquals(localAsString, new String(localContents.toByteArray()));
+		secondRemote.setLastModified(newLastModified);
+		this.inboundMGetRecursive.send(new GenericMessage<Object>("*"));
+		this.output.receive(0);
+		localContents = new ByteArrayOutputStream();
+		FileUtils.copyFile(secondTarget, localContents);
+		assertEquals("junk", new String(localContents.toByteArray()));
+		// restore the remote file contents
+		FileUtils.copyInputStreamToFile(new ByteArrayInputStream(localAsString.getBytes()), secondRemote);
 	}
 
 	private long setModifiedOnSource1() {

spring-integration-gemfire/src/test/resources/log4j.xml

@@ -25,4 +25,4 @@
 		<appender-ref ref="console" />
 	</root>
 
-</log4j:configuration>
+</log4j:configuration>

spring-integration-scripting/src/test/resources/log4j.xml

@@ -26,4 +26,4 @@
 		<appender-ref ref="console" />
 	</root>
 
-</log4j:configuration>
+</log4j:configuration>

spring-integration-sftp/src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml

@@ -53,6 +53,7 @@
 							  command="mget"
 							  expression="payload"
 							  command-options="-R -P"
+							  mode="REPLACE_IF_MODIFIED"
 							  filter="dotStarDotTxtFilter"
 							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"