Skip to content
Browse files

added initial version open source version

  • Loading branch information...
0 parents commit 8d0a03a2750a2a6e9f9d43af02e3fd98840d5072 @cosmin cosmin committed Jul 3, 2012
3 .gitignore
@@ -0,0 +1,3 @@
+/target
+.idea
+*.iml
41 LICENSE
@@ -0,0 +1,41 @@
+Copyright (c) 2012 Simple Finance Technology Corp.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+
+
+GPG File Decryption code based the KeyBasedFileProcessor example code
+from bouncycastle.org, originally distributed under the following
+license:
+
+Copyright (c) 2000 - 2011 The Legion Of The Bouncy Castle (http://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
65 README.md
@@ -0,0 +1,65 @@
+# sftp-fetch
+
+Download new files from an SFTP server and publish them to a queue,
+using S3 to hold the data.
+
+## Configuration
+
+First, create a properties file that looks like
+
+```
+sftp.hostname=<the.sftp.hostname>
+sftp.username=<the.sftp.username>
+sftp.password=<the.sftp.password>
+sftp.folder=<the.sftp.folder.to.download.from>
+
+rabbit.hostname=<the.rabbit.hostname>
+rabbit.port=5672
+rabbit.username=<the.rabbit.username>
+rabbit.password=<the.rabbit.password>
+rabbit.vhost=<the.rabbit.vhost>
+rabbit.exchange=<the.rabbit.exchange>
+
+s3.bucket=<s3.bucket>
+fetch.days=<number.of.days.to.fetch>
+```
+
+## Usage
+
+### The Basics
+
+Once you have created the appropriate config files you can invoke
+
+```
+java -jar sftp-fetch.jar -c </path/to/properties/file> -n
+```
+
+This will show you what files it would process without actually doing
+it. Run without `-n` to actually process. Files that have already been
+processed will be skipped unless you run with `--overwrite`
+
+### Files that match a pattern
+
+Optionally you can restrict operations only to files that match a
+certain pattern using `-p`
+
+```
+java -jar sftp-fetch.jar -c </path/to/properties/file> -r <routingkey> -p '.*SomeFileName.*'
+```
+
+### Specify the routing key
+
+You can specify a routing key using `--routing-key` at the command
+line or `rabbit.routingkey` in the property file.
+
+### GPG decryption
+
+If you are fetching GPG encrypted files from SFTP you can optionally
+have `sft-fetch` decrypt them by supplying the path to the GPG private
+key in the property file using
+
+```
+decryption.key.path=</path/to/pgp/private/key>
+```
+
+Currently encrypted private keys are not supported.
155 pom.xml
@@ -0,0 +1,155 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.simple</groupId>
+ <artifactId>sftp-fetch</artifactId>
+ <packaging>jar</packaging>
+ <version>1.0.1-SNAPSHOT</version>
+
+ <name>bancorp-sftp</name>
+ <url>https://github.com/SimpleFinance/sftp-fetch</url>
+ <description>Download new files from an SFTP server and publish them to a queue, using S3 to hold the data
+ </description>
+
+ <developers>
+ <developer>
+ <name>Cosmin Stejerean</name>
+ <email>cosmin@offbytwo.com</email>
+ </developer>
+ </developers>
+
+ <licenses>
+ <license>
+ <name>Apache Public License (v2.0)</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+ </license>
+ </licenses>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.rabbitmq</groupId>
+ <artifactId>amqp-client</artifactId>
+ <version>2.7.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.48</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.3.11</version>
+ </dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>2.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpg-jdk16</artifactId>
+ <version>1.46</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.0</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.8.1</version>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.1.2</version>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>1.6</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>standalone</shadedClassifierName>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>com.simple.sftpfetch.App</mainClass>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/*.INF</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>2.4</version>
+ <configuration>
+ <archive>
+ <manifest>
+ <addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+ </manifest>
+ </archive>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
223 src/main/java/com/simple/sftpfetch/App.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch;
+
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.SftpException;
+import com.rabbitmq.client.ConnectionFactory;
+import com.simple.sftpfetch.decrypt.FileDecrypter;
+import com.simple.sftpfetch.decrypt.NoopDecrypter;
+import com.simple.sftpfetch.decrypt.PGPFileDecrypter;
+import com.simple.sftpfetch.publish.RabbitClient;
+import com.simple.sftpfetch.publish.RabbitConnectionInfo;
+import com.simple.sftpfetch.publish.S3;
+import com.simple.sftpfetch.publish.SuppliedAWSCredentials;
+import com.simple.sftpfetch.sftp.SftpClient;
+import com.simple.sftpfetch.sftp.SftpConnectionInfo;
+import org.apache.commons.cli.*;
+
+import java.io.*;
+import java.security.NoSuchProviderException;
+import java.security.Security;
+import java.util.List;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
+import static java.util.Arrays.asList;
+
+/**
+ * Main application entry point
+ */
+public class App {
+ public static final String FETCH_DAYS = "fetch.days";
+ public static final Pattern MATCH_EVERYTHING = Pattern.compile(".*");
+ private SftpClient sftp;
+ private S3 s3;
+ private RabbitClient rabbit;
+ private FileDecrypter decrypter;
+ private PrintStream out;
+
+ public App(SftpClient sftpClient, S3 s3, RabbitClient rabbitClient, FileDecrypter decrypter, PrintStream out) {
+ this.sftp = sftpClient;
+ this.s3 = s3;
+ this.rabbit = rabbitClient;
+ this.decrypter = decrypter;
+ this.out = out;
+ }
+
+ private static S3 s3FromProperties(Properties properties) {
+ String s3Bucket = properties.getProperty("s3.bucket");
+ final String awsAccessKey = properties.getProperty("s3.access.key", "");
+ final String awsSecretKey = properties.getProperty("s3.secret.key", "");
+
+ AmazonS3Client client;
+ if (awsAccessKey.isEmpty() || awsSecretKey.isEmpty()) {
+ client = new AmazonS3Client();
+ } else {
+ client = new AmazonS3Client(new SuppliedAWSCredentials(awsAccessKey, awsSecretKey));
+ }
+
+ return new S3(client, s3Bucket);
+ }
+
+
+ /**
+ * A simpler entry point with sane defaults, used by the tests
+ *
+ * @param routingKey the routing key to use for publishing rabbit messages
+ * @param daysToFetch the number of days back to fetch from SFTP
+ *
+ * @throws IOException
+ * @throws NoSuchProviderException
+ * @throws SftpException
+ */
+ void run(String routingKey, int daysToFetch) throws IOException, NoSuchProviderException, SftpException {
+ run(routingKey, daysToFetch, App.MATCH_EVERYTHING, false, false);
+ }
+
+ /**
+ * Run the application
+ *
+ * @param routingKey the routing key to use for publishing rabbit messages
+ * @param daysToFetch the number of days back to fetch from SFTP
+ * @param pattern a regular expression pattern that files must match to be processed
+ * @param noop if true do not actually modify anything
+ * @param overwrite re-publish previously seen files
+ *
+ * @throws SftpException
+ * @throws IOException
+ * @throws NoSuchProviderException
+ */
+ public void run(String routingKey, int daysToFetch, Pattern pattern, boolean noop, boolean overwrite) throws SftpException, IOException, NoSuchProviderException {
+ for (String filename : sftp.getFilesNewerThan(daysToFetch, pattern)) {
+ if (s3.keyExists(filename)) {
+ out.println("Previously seen: " + filename);
+ if (!overwrite) {
+ continue;
+ }
+ }
+ if (noop) {
+ out.println("Would process: " + filename);
+ } else {
+ File downloaded = sftp.downloadFile(filename);
+ File toUpload = decrypter.decryptFile(downloaded);
+ s3.upload(filename, toUpload);
+ rabbit.publishURL(routingKey, s3.getURLFor(filename));
+ out.println("Processed: " + filename);
+ }
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider());
+
+ Options options = getOptions();
+
+ List<String> requiredProperties = asList("c");
+
+
+ CommandLineParser parser = new PosixParser();
+ try {
+ CommandLine commandLine = parser.parse(options, args);
+ if (commandLine.hasOption("h")) {
+ printUsage(options);
+ System.exit(0);
+ }
+
+ for (String opt : requiredProperties) {
+ if (!commandLine.hasOption(opt)) {
+ System.err.println("The option: " + opt + " is required.");
+ printUsage(options);
+ System.exit(1);
+ }
+ }
+
+ Pattern pattern;
+ if (commandLine.hasOption("p")) {
+ pattern = Pattern.compile(commandLine.getOptionValue("p"));
+ } else {
+ pattern = MATCH_EVERYTHING;
+ }
+
+ String filename = commandLine.getOptionValue("c");
+ Properties properties = new Properties();
+ try {
+ InputStream stream = new FileInputStream(new File(filename));
+ properties.load(stream);
+ } catch (IOException ioe) {
+ System.err.println("Unable to read properties from: " + filename);
+ System.exit(2);
+ }
+
+ String routingKey = "";
+ if (commandLine.hasOption("r")) {
+ routingKey = commandLine.getOptionValue("r");
+ } else if (properties.containsKey("rabbit.routingkey")) {
+ routingKey = properties.getProperty("rabbit.routingkey");
+ }
+
+ int daysToFetch;
+ if (commandLine.hasOption("d")) {
+ daysToFetch = Integer.valueOf(commandLine.getOptionValue("d"));
+ } else {
+ daysToFetch = Integer.valueOf(properties.getProperty(FETCH_DAYS));
+ }
+
+ FileDecrypter decrypter = null;
+ if (properties.containsKey("decryption.key.path")) {
+ decrypter = new PGPFileDecrypter(new File(properties.getProperty("decryption.key.path")));
+ } else {
+ decrypter = new NoopDecrypter();
+ }
+
+ SftpClient sftpClient = new SftpClient(new JSch(), new SftpConnectionInfo(properties));
+ try {
+ App app = new App(sftpClient,
+ s3FromProperties(properties),
+ new RabbitClient(new ConnectionFactory(), new RabbitConnectionInfo(properties)),
+ decrypter,
+ System.out);
+ app.run(routingKey, daysToFetch, pattern, commandLine.hasOption("n"), commandLine.hasOption("o"));
+ } finally {
+ sftpClient.close();
+ }
+ System.exit(0);
+ } catch (UnrecognizedOptionException uoe) {
+ System.err.println(uoe.getMessage());
+ printUsage(options);
+ System.exit(10);
+ }
+ }
+
+ public static Options getOptions() {
+ Options options = new Options();
+ options.addOption("n", "noop", false, "Don't download or publish anything, simply mention what would be done");
+ options.addOption("o", "overwrite", false, "Re-publish previously seen files");
+ options.addOption("p", "pattern", true, "Only download files that match this pattern.");
+ options.addOption("r", "routing-key", true, "Routing key for posting messages");
+ options.addOption("c", "config", true, "Properties file containing configuration options");
+ options.addOption("d", "days", true, "Download files newer than this many days ago");
+ options.addOption("h", "help", false, "Show this screen");
+ return options;
+ }
+
+ public static void printUsage(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("java -jar sftp-fetch.jar [options]", options);
+ }
+}
37 src/main/java/com/simple/sftpfetch/decrypt/FileDecrypter.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.decrypt;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchProviderException;
+
+/**
+ * A simple interface for decrypting files
+ */
+public interface FileDecrypter {
+ /**
+ * Decrypt the given file
+ *
+ * @param input an encrypted file
+ * @return the decrypted file
+ *
+ * @throws IOException
+ * @throws NoSuchProviderException
+ */
+ File decryptFile(File input) throws IOException, NoSuchProviderException;
+}
40 src/main/java/com/simple/sftpfetch/decrypt/NoopDecrypter.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.decrypt;
+
+import java.io.File;
+import java.io.IOException;
+import java.security.NoSuchProviderException;
+
+/**
+ * Does not actually decrypt files, it merely returns the input
+ */
+public class NoopDecrypter implements FileDecrypter {
+ /**
+ * Pretend to decrypt the given file, but merely return the input
+ *
+ * @param input an encrypted file
+ * @return the decrypted file
+ * @throws java.io.IOException
+ * @throws java.security.NoSuchProviderException
+ *
+ */
+ @Override
+ public File decryptFile(File input) throws IOException, NoSuchProviderException {
+ return input;
+ }
+}
178 src/main/java/com/simple/sftpfetch/decrypt/PGPFileDecrypter.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *
+ * Based on the org.bouncycastle.openpgp.examples.KeyBasedFileProcessor
+ * example code, distributed under the following license license:
+ *
+ * Copyright (c) 2000 - 2011 The Legion Of The Bouncy Castle (http://www.bouncycastle.org)
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
+ *
+ */
+
+package com.simple.sftpfetch.decrypt;
+
+import org.bouncycastle.openpgp.*;
+import org.bouncycastle.util.io.Streams;
+
+import java.io.*;
+import java.security.NoSuchProviderException;
+import java.util.Iterator;
+
+/**
+ * PGP file decrypter based on {@link org.bouncycastle.openpgp.examples.KeyBasedFileProcessor}
+ */
+public class PGPFileDecrypter implements FileDecrypter {
+
+ private PGPSecretKeyRingCollection pgpSec;
+
+ /**
+ * Initialize the PGPFileDecrypter with the private key from the given file
+ *
+ * @param key file containing unencrypted private key
+ *
+ * @throws IOException
+ * @throws PGPException
+ */
+ public PGPFileDecrypter(File key) throws IOException, PGPException {
+ InputStream keyIn = new FileInputStream(key);
+ try {
+ this.pgpSec = new PGPSecretKeyRingCollection(PGPUtil.getDecoderStream(keyIn));
+ } finally {
+ keyIn.close();
+ }
+ }
+
+ /**
+ * Decrypt the given file and return the resulting File
+ *
+ * @param input an encrypted file
+ *
+ * @return the decrypted file
+ * @throws IOException
+ * @throws NoSuchProviderException
+ */
+ @Override
+ public File decryptFile(File input) throws IOException, NoSuchProviderException {
+ InputStream in = new BufferedInputStream(new FileInputStream(input));
+ File out = File.createTempFile("message", ".txt");
+ OutputStream outStream = new BufferedOutputStream(new FileOutputStream(out));
+ try {
+ decryptFile(in, outStream);
+ } finally {
+ outStream.close();
+ in.close();
+ }
+
+ return out;
+ }
+
+ private void decryptFile(InputStream in, OutputStream outputStream) throws IOException, NoSuchProviderException {
+ in = PGPUtil.getDecoderStream(in);
+
+ try {
+ PGPEncryptedDataList enc = getEncryptedDataList(in);
+
+ Iterator it = enc.getEncryptedDataObjects();
+ PGPPrivateKey sKey = null;
+ PGPPublicKeyEncryptedData pbe = null;
+
+ while (sKey == null && it.hasNext()) {
+ pbe = (PGPPublicKeyEncryptedData) it.next();
+ sKey = getPrivateKey(sKey, pbe);
+ }
+
+ if (sKey == null) {
+ throw new IllegalArgumentException("secret key for message not found.");
+ }
+
+ InputStream clear = pbe.getDataStream(sKey, "BC");
+ Object message = new PGPObjectFactory(clear).nextObject();
+
+ if (message instanceof PGPCompressedData) {
+ PGPCompressedData cData = (PGPCompressedData) message;
+ PGPObjectFactory pgpFact = new PGPObjectFactory(cData.getDataStream());
+ message = pgpFact.nextObject();
+ }
+
+ if (message instanceof PGPLiteralData) {
+ PGPLiteralData ld = (PGPLiteralData) message;
+ Streams.pipeAll(ld.getInputStream(), outputStream);
+ } else if (message instanceof PGPOnePassSignatureList) {
+ throw new PGPException("encrypted message contains a signed message - not literal data.");
+ } else {
+ throw new PGPException("message is not a simple encrypted file - type unknown.");
+ }
+
+ if (pbe.isIntegrityProtected() && !pbe.verify()) {
+ throw new PGPException("message failed integrity check");
+ }
+ } catch (PGPException e) {
+ System.err.println(e);
+ if (e.getUnderlyingException() != null) {
+ e.getUnderlyingException().printStackTrace();
+ }
+ }
+ }
+
+ private PGPEncryptedDataList getEncryptedDataList(InputStream in) throws IOException {
+ PGPObjectFactory pgpF = new PGPObjectFactory(in);
+ PGPEncryptedDataList enc;
+ Object o = pgpF.nextObject(); // the first object might be a PGP marker packet.
+ if (o instanceof PGPEncryptedDataList) {
+ enc = (PGPEncryptedDataList) o;
+ } else {
+ enc = (PGPEncryptedDataList) pgpF.nextObject();
+ }
+ return enc;
+ }
+
+ /**
+ * Get the matching PGPPrivateKey for the given public key encrypted data
+ *
+ * @param sKey private key information
+ * @param pbe public key encrypted data
+ *
+ * @return the matching PGPPrivateKey
+ *
+ * @throws PGPException
+ * @throws NoSuchProviderException
+ */
+ private PGPPrivateKey getPrivateKey(PGPPrivateKey sKey, PGPPublicKeyEncryptedData pbe) throws PGPException, NoSuchProviderException {
+ PGPSecretKey pgpSecKey = pgpSec.getSecretKey(pbe.getKeyID());
+ if (pgpSecKey != null) {
+ sKey = pgpSecKey.extractPrivateKey(new char[]{}, "BC");
+ }
+ return sKey;
+ }
+}
78 src/main/java/com/simple/sftpfetch/publish/RabbitClient.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.publish;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * A very simple interface for publishing plain-text messages to RabbitMQ using a direct exchange
+ */
+public class RabbitClient {
+ public static final String EXCHANGE_TYPE = "direct";
+ public static final String ENCODING = "UTF8";
+ public static final String CONTENT_TYPE = "text/plain";
+
+ AMQP.BasicProperties amqpProperties;
+ private Channel channel;
+ private String exchange;
+
+ /**
+ * Initialize the RabbitClient, establish a connection and declare the exchange
+ *
+ * @param factory the RabbitMQ ConnectionFactory
+ * @param connectionInfo a bean containing the necessary connection information
+ *
+ * @throws NoSuchAlgorithmException
+ * @throws KeyManagementException
+ * @throws URISyntaxException
+ * @throws IOException
+ */
+ public RabbitClient(ConnectionFactory factory, RabbitConnectionInfo connectionInfo) throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException, IOException {
+ factory.setHost(connectionInfo.getHostname());
+ factory.setPort(connectionInfo.getPort());
+ factory.setUsername(connectionInfo.getUsername());
+ factory.setPassword(connectionInfo.getPassword());
+ factory.setVirtualHost(connectionInfo.getVhost());
+ factory.setConnectionTimeout(connectionInfo.getTimeout());
+ Connection conn = factory.newConnection();
+ exchange = connectionInfo.getExchange();
+ channel = conn.createChannel();
+ channel.exchangeDeclare(exchange, EXCHANGE_TYPE, true);
+ this.amqpProperties = new AMQP.BasicProperties.Builder().contentType(CONTENT_TYPE).deliveryMode(2).build();
+ }
+
+ /**
+ * Publish the given URL as a plain-text message with the given routing key
+ *
+ * @param routingKey the routing key to use
+ * @param url the URL to publish
+ *
+ * @throws IOException
+ */
+ public void publishURL(String routingKey, URL url) throws IOException {
+ channel.basicPublish(exchange, routingKey, amqpProperties, url.toString().getBytes(ENCODING));
+ }
+}
100 src/main/java/com/simple/sftpfetch/publish/RabbitConnectionInfo.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.publish;
+
+import java.util.Properties;
+
+public class RabbitConnectionInfo {
+ public static final int DEFAULT_TIMEOUT = 5000;
+ public static final int DEFAULT_PORT = 5672;
+ private final String vhost;
+ private String exchange;
+ private int timeout;
+ private final String username;
+ private final String password;
+ private String hostname;
+ private int port;
+
+ /**
+ * Initialize using data from the supplied Properties files, using the following required keys
+ *
+ * <ul>
+ * <li>rabbit.password</li>
+ * <li>rabbit.hostname</li>
+ * <li>rabbit.vhost</li>
+ * <li>rabbit.exchange</li>
+ * <li>rabbit.username</li>
+ * </ul>
+ *
+ * and the following optional keys
+ *
+ * <ul>
+ * <li>rabbit.port</li>
+ * <li>rabbit.connection.timeout</li>
+ * </ul>
+ *
+ * @param properties Properties containing the above keys
+ */
+ public RabbitConnectionInfo(Properties properties) {
+ this(properties.getProperty("rabbit.hostname"),
+ Integer.valueOf(properties.getProperty("rabbit.port", String.valueOf(DEFAULT_PORT))),
+ properties.getProperty("rabbit.vhost"),
+ properties.getProperty("rabbit.exchange"),
+ Integer.valueOf(properties.getProperty("rabbit.connection.timeout", String.valueOf(DEFAULT_TIMEOUT))),
+ properties.getProperty("rabbit.username"),
+ properties.getProperty("rabbit.password")
+ );
+ }
+
+ public RabbitConnectionInfo(String hostname, int port, String vhost, String exchange, int timeout, String username, String password) {
+ this.hostname = hostname;
+ this.port = port;
+ this.vhost = vhost;
+ this.exchange = exchange;
+ this.timeout = timeout;
+ this.username = username;
+ this.password = password;
+ }
+
+ public String getExchange() {
+ return exchange;
+ }
+
+ public int getTimeout() {
+ return timeout;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public String getVhost() {
+ return vhost;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+}
105 src/main/java/com/simple/sftpfetch/publish/S3.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.publish;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3Client;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+/**
+ * A simple client for uploading files to S3
+ */
+public class S3 {
+ private AmazonS3Client s3;
+ private String bucket;
+ private String location;
+
+ /**
+ * Initialize the client using the supplied {@link AmazonS3Client} and bucket name.
+ * This will verify if the bucket exists and retrieve the location of the given bucket to be used later.
+ *
+ * @param s3 the AmazonS3Client to use
+ * @param bucket the name of the bucket to use
+ */
+ public S3(AmazonS3Client s3, String bucket) {
+ this.bucket = bucket;
+ this.s3 = s3;
+ if (!this.s3.doesBucketExist(bucket)) {
+ throw new AmazonServiceException("Bucket does not exist: " + bucket);
+ }
+ this.location = s3.getBucketLocation(bucket);
+ }
+
+ /**
+ * Check if the given key exists
+ *
+ * @param key the key to check
+ * @return true if the key exists, false otherwise
+ */
+ public boolean keyExists(String key){
+ try {
+ s3.getObjectMetadata(bucket, key);
+ } catch (AmazonServiceException ase) {
+ if (ase.getStatusCode() == 404) {
+ return false;
+ } else {
+ throw ase;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Upload the given file using the given key
+ *
+ * @param key the key to use
+ * @param toUpload to file to upload
+ */
+ public void upload(String key, File toUpload) {
+ s3.putObject(bucket, key, toUpload);
+ }
+
+ /**
+ * Get the hostname to use in creating HTTP URLs for S3 objects
+ *
+ * @return the hostname
+ */
+ public String getS3HostName() {
+ String prefix;
+ if (location.equalsIgnoreCase("US")) {
+ prefix = "s3";
+ } else {
+ prefix = "s3-" + this.location;
+ }
+ return prefix + ".amazonaws.com";
+ }
+
+ /**
+ * Create an absolute HTTP URL for the object at the given key
+ *
+ * @param key the key
+ *
+ * @return the absolute HTTP URL
+ * @throws MalformedURLException
+ */
+ public URL getURLFor(String key) throws MalformedURLException {
+ return new URL("https://" + this.getS3HostName() + "/" + this.bucket + "/" + key);
+ }
+}
39 src/main/java/com/simple/sftpfetch/publish/SuppliedAWSCredentials.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.publish;
+
+import com.amazonaws.auth.AWSCredentials;
+
+public class SuppliedAWSCredentials implements AWSCredentials {
+ private final String awsAccessKey;
+ private final String awsSecretKey;
+
+ public SuppliedAWSCredentials(String awsAccessKey, String awsSecretKey) {
+ this.awsAccessKey = awsAccessKey;
+ this.awsSecretKey = awsSecretKey;
+ }
+
+ @Override
+ public String getAWSAccessKeyId() {
+ return awsAccessKey;
+ }
+
+ @Override
+ public String getAWSSecretKey() {
+ return awsSecretKey;
+ }
+}
79 src/main/java/com/simple/sftpfetch/sftp/PasswordBasedAuthentication.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.sftp;
+
+import com.jcraft.jsch.UserInfo;
+
+/**
+ * An implementation of {@link UserInfo} using a supplied password
+ */
+public class PasswordBasedAuthentication implements UserInfo {
+ private String password;
+
+ /**
+ * @param password the password to use
+ */
+ public PasswordBasedAuthentication(String password) {
+ this.password = password;
+ }
+
+ @Override
+ public String getPassphrase() {
+ return null;
+ }
+
+ @Override
+ public String getPassword() {
+ return this.password;
+ }
+
+ @Override
+ public boolean promptPassword(String message) {
+ return true;
+ }
+
+ @Override
+ public boolean promptPassphrase(String message) {
+ return false;
+ }
+
+ @Override
+ public boolean promptYesNo(String message) {
+ return false;
+ }
+
+ @Override
+ public void showMessage(String message) {
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ PasswordBasedAuthentication that = (PasswordBasedAuthentication) o;
+
+ if (password != null ? !password.equals(that.password) : that.password != null) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return password != null ? password.hashCode() : 0;
+ }
+}
136 src/main/java/com/simple/sftpfetch/sftp/SftpClient.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.sftp;
+
+import com.jcraft.jsch.*;
+import org.joda.time.DateTime;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+/**
+ * A wrapper for fetching files from SFTP
+ */
+public class SftpClient {
+ private JSch jsch;
+ private ChannelSftp sftp;
+ private Session session;
+ private String downloadFrom;
+
+ /**
+ * Initialize using the supplied {@link JSch} client SftpConnectionInfo
+ *
+ * @param jsch the JSch client
+ * @param connectionInfo the connection info bean
+ *
+ * @throws JSchException
+ */
+ public SftpClient(JSch jsch, SftpConnectionInfo connectionInfo) throws JSchException {
+ this.session = jsch.getSession(connectionInfo.getUsername(), connectionInfo.getHostname(), connectionInfo.getPort());
+ jsch.setKnownHosts(new File(System.getProperty("user.home"), ".ssh/known_hosts").getAbsolutePath());
+ this.downloadFrom = connectionInfo.getDownloadFrom();
+ this.session.setUserInfo(new PasswordBasedAuthentication(connectionInfo.getPassword()));
+ this.session.connect(connectionInfo.getTimeout());
+ this.sftp = (ChannelSftp) session.openChannel("sftp");
+ this.sftp.connect(connectionInfo.getTimeout());
+ }
+
+ /**
+ * Disconnect the sftp connection and session
+ */
+ public void close() {
+ this.sftp.disconnect();
+ this.session.disconnect();
+ }
+
+ /**
+ * Get all the file names newer than days ago
+ *
+ * @param days the number of days
+ *
+ * @return a set of file names
+ * @throws SftpException
+ */
+ public Set<String> getFilesNewerThan(int days) throws SftpException {
+ return getFilesNewerThan(days, Pattern.compile(".*"));
+ }
+
+ /**
+ * Get all the file names newer than days ago that match the given pattern
+ *
+ * @param days the number of days
+ * @param pattern the pattern to match
+ *
+ * @return a set of file names
+ * @throws SftpException
+ */
+ public Set<String> getFilesNewerThan(int days, Pattern pattern) throws SftpException {
+ long since = new DateTime().minusDays(days).getMillis() / 1000;
+
+ Set<String> files = new HashSet<String>();
+ for (Object obj : sftp.ls(downloadFrom)) {
+ if (obj instanceof com.jcraft.jsch.ChannelSftp.LsEntry) {
+ ChannelSftp.LsEntry entry = (ChannelSftp.LsEntry) obj;
+ if (entry.getAttrs().getMTime() >= since) {
+ if (pattern.matcher(entry.getFilename()).matches()) {
+ files.add(entry.getFilename());
+ }
+ }
+ }
+ }
+ return files;
+ }
+
+ /**
+ * Create a path from the given filename and the folder we are downloading from
+ *
+ * @param filename the filename
+ * @return the full path on the remote server
+ */
+ String pathForFilename(String filename) {
+ if (downloadFrom == null || downloadFrom.length() == 0) {
+ return filename;
+ } else {
+ if (downloadFrom.endsWith("/")) {
+ return downloadFrom + filename;
+ } else {
+ return downloadFrom + "/" + filename;
+ }
+ }
+ }
+
+ /**
+ * Download the given file
+ *
+ * @param filename a filename (relative to the downloadFrom folder)
+ * @return a temporary file storing the downloaded contents
+ *
+ * @throws SftpException
+ * @throws IOException
+ */
+ public File downloadFile(String filename) throws SftpException, IOException {
+ File tempFile = File.createTempFile("sftp", ".download");
+ FileOutputStream fileOutputStream = new FileOutputStream(tempFile);
+ this.sftp.get(pathForFilename(filename), fileOutputStream);
+ fileOutputStream.flush();
+ return tempFile;
+ }
+}
88 src/main/java/com/simple/sftpfetch/sftp/SftpConnectionInfo.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2012 Simple Finance Technology Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.simple.sftpfetch.sftp;
+
+import java.util.Properties;
+
+/**
+ * Hold all the necessary information for the SFTP connection
+ */
+public class SftpConnectionInfo {
+ public static final int DEFAULT_PORT = 22;
+ public static final int DEFAULT_TIMEOUT = 5000;
+ private String username;
+ private String password;
+ private String hostname;
+ private int port;
+ private int timeout;
+ private String downloadFrom;
+
+ /**
+ * Initialize from the given Properties, containing the following keys
+ *
+ * <ul>
+ * <li>sftp.username</li>
+ * <li>sftp.password</li>
+ * <li>sftp.folder</li>
+ * <li>sftp.hostname</li>
+ * <li>sftp.port</li>
+ * <li>sftp.timeout</li>
+ * </ul>
+ *
+ * @param properties the given properties
+ */
+ public SftpConnectionInfo(Properties properties) {
+ this(properties.getProperty("sftp.username"),
+ properties.getProperty("sftp.password"),
+ properties.getProperty("sftp.folder", ""), properties.getProperty("sftp.hostname"),
+ Integer.valueOf(properties.getProperty("sftp.port", String.valueOf(DEFAULT_PORT))),
+ Integer.valueOf(properties.getProperty("sftp.timeout", String.valueOf(DEFAULT_TIMEOUT))));
+ }
+
+ public SftpConnectionInfo(String username, String password, String downloadFrom, String hostname, int port, int timeout) {
+ this.username = username;
+ this.password = password;
+ this.hostname = hostname;
+ this.downloadFrom = downloadFrom;
+ this.port = port;
+ this.timeout = timeout;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public int getPort() {
+ return port;
+ }
+
+ public int getTimeout() {
+ return timeout;
+ }
+
+ public String getDownloadFrom() {
+ return downloadFrom;
+ }
+}
184 src/test/java/com/simple/sftpfetch/AppTest.java
@@ -0,0 +1,184 @@
+package com.simple.sftpfetch;
+
+import com.jcraft.jsch.SftpException;
+import com.simple.sftpfetch.decrypt.PGPFileDecrypter;
+import com.simple.sftpfetch.publish.RabbitClient;
+import com.simple.sftpfetch.publish.S3;
+import com.simple.sftpfetch.sftp.SftpClient;
+import org.apache.commons.cli.Options;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.util.HashSet;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.*;
+
+public class AppTest {
+
+ private String routing_key = "routingkey";
+ private SftpClient sftpClient = mock(SftpClient.class);
+ private S3 s3 = mock(S3.class);
+ private RabbitClient rabbitClient = mock(RabbitClient.class);
+ private PGPFileDecrypter decrypter = mock(PGPFileDecrypter.class);
+ private App app = new App(sftpClient, s3, rabbitClient, decrypter, mock(PrintStream.class));
+ private String filename = "foo";
+ private URL url;
+
+ @Before
+ public void setUp() throws Exception {
+ createOneRemoteFile();
+ url = new URL("http://google.com");
+ when(s3.getURLFor(filename)).thenReturn(url);
+ }
+
+ @Test
+ public void shouldContainShortOptions() {
+ Options options = App.getOptions();
+ for (String option : asList("n", "o", "c", "d", "h")) {
+ assertNotNull("Cannot find option: " + option, options.getOption(option));
+ }
+ }
+
+ @Test
+ public void shouldContainLongOptions() {
+ Options options = App.getOptions();
+ for (String option : asList("noop", "overwrite", "config", "days", "help")) {
+ assertNotNull("Cannot find option: " + option, options.getOption(option));
+ }
+ }
+
+ @Test
+ public void shouldAskForLastDaysWorthOfOptiosn() throws Exception {
+ int days = 4;
+ createOneRemoteFile();
+
+ app.run(routing_key, days);
+
+ verify(sftpClient).getFilesNewerThan(days, App.MATCH_EVERYTHING);
+ }
+
+ @Test
+ public void shouldCheckIfFilesExistInS3() throws Exception {
+ createOneRemoteFile();
+
+ invokeTheDefault();
+
+ verify(s3).keyExists(filename);
+ }
+
+ @Test
+ public void shouldSkipOverFilesThatExistInS3() throws Exception {
+ createOneRemoteFile();
+ theFileExistsInS3();
+
+ invokeTheDefault();
+
+ verifyFileNotUploaded();
+ }
+
+ @Test
+ public void shouldUploadFilesThatDoNotExistInS3() throws Exception {
+ createOneRemoteFile();
+ theFileDoesNotExistInS3();
+
+ invokeTheDefault();
+
+ verifyFileUploaded();
+ }
+
+ @Test
+ public void shouldUploadFilesThatExistIfOverwrite() throws Exception {
+ createOneRemoteFile();
+ theFileExistsInS3();
+
+ invokeWithOverwrite();
+
+ verifyFileUploaded();
+ }
+
+ @Test
+ public void shouldNotUploadFilesThatDoNotExistIfNoop() throws Exception {
+ createOneRemoteFile();
+ theFileDoesNotExistInS3();
+
+ invokeWithNoop();
+
+ verifyFileNotUploaded();
+ }
+
+ @Test
+ public void shouldPublishUrlsToRabbitForNewFiles() throws Exception {
+ createOneRemoteFile();
+ theFileDoesNotExistInS3();
+
+ invokeTheDefault();
+
+ verifyRabbitDidPublish();
+ }
+
+ @Test
+ public void shouldNotPublishUrlsToRabbitIfNoop() throws Exception {
+ createOneRemoteFile();
+ theFileDoesNotExistInS3();
+
+ invokeWithNoop();
+
+ verifyRabbitDidNotPublish();
+ }
+
+ @Test
+ public void shouldPublishUrlsToRabbitForExistingFilesIfOverwrite() throws Exception {
+ createOneRemoteFile();
+ theFileExistsInS3();
+
+ invokeWithOverwrite();
+
+ verifyRabbitDidPublish();
+ }
+
+ private void verifyRabbitDidPublish() throws IOException {
+ verify(rabbitClient).publishURL(routing_key, url);
+ }
+
+ private void verifyRabbitDidNotPublish() {
+ verifyZeroInteractions(rabbitClient);
+ }
+
+ private void invokeWithOverwrite() throws Exception {
+ app.run(routing_key, 1, App.MATCH_EVERYTHING, false, true);
+ }
+
+ private void invokeWithNoop() throws Exception {
+ app.run(routing_key, 1, App.MATCH_EVERYTHING, true, false);
+ }
+
+ private void invokeTheDefault() throws Exception {
+ app.run(routing_key, 1);
+ }
+
+ private void verifyFileNotUploaded() {
+ verify(s3, never()).upload(eq(filename), any(File.class));
+ }
+
+ private void verifyFileUploaded() {
+ verify(s3).upload(eq(filename), any(File.class));
+ }
+
+ private void theFileDoesNotExistInS3() {
+ when(s3.keyExists(filename)).thenReturn(false);
+ }
+
+ private void theFileExistsInS3() {
+ when(s3.keyExists(filename)).thenReturn(true);
+ }
+
+ private void createOneRemoteFile() throws SftpException {
+ when(sftpClient.getFilesNewerThan(anyInt(), eq(App.MATCH_EVERYTHING))).thenReturn(new HashSet<String>(asList(filename)));
+ }
+}
59 src/test/java/com/simple/sftpfetch/publish/RabbitClientTest.java
@@ -0,0 +1,59 @@
+package com.simple.sftpfetch.publish;
+
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.net.URL;
+
+import static junit.framework.Assert.assertEquals;
+import static org.mockito.Mockito.*;
+
+public class RabbitClientTest {
+
+ private ConnectionFactory factory = mock(ConnectionFactory.class);
+ private Connection connection = mock(Connection.class);
+ private Channel channel = mock(Channel.class);
+ public static final String CONN = "amqp://username:password@rabbit.example.com:5672/the/vhost";
+ public static final String EXCHANGE = "foobar";
+ public static final String ROUTING_KEY = "route.this";
+ private RabbitConnectionInfo connectionInfo = mock(RabbitConnectionInfo.class);
+
+ @Before
+ public void setUp() throws Exception {
+ when(connectionInfo.getExchange()).thenReturn(EXCHANGE);
+ when(factory.newConnection()).thenReturn(connection);
+ when(connection.createChannel()).thenReturn(channel);
+ }
+
+ @Test
+ public void shouldDeclareExchnage() throws Exception {
+ new RabbitClient(factory, connectionInfo);
+ verify(channel).exchangeDeclare(EXCHANGE, RabbitClient.EXCHANGE_TYPE, true);
+ }
+
+ @Test
+ public void shouldEstablishConnectionTimeout() throws Exception {
+ int timeout = 1337;
+ when(connectionInfo.getTimeout()).thenReturn(timeout);
+ new RabbitClient(factory, connectionInfo);
+ verify(factory).setConnectionTimeout(timeout);
+ }
+
+ @Test
+ public void shouldCreateCorrectProperties() throws Exception {
+ RabbitClient client = new RabbitClient(factory, connectionInfo);
+ assertEquals("text/plain", client.amqpProperties.getContentType());
+ assertEquals(2, (int) client.amqpProperties.getDeliveryMode());
+ }
+
+ @Test
+ public void shouldPublishUrlAsUTF8() throws Exception {
+ RabbitClient client = new RabbitClient(factory, connectionInfo);
+ URL url = new URL("http://google.com");
+ client.publishURL(ROUTING_KEY, url);
+ verify(channel).basicPublish(EXCHANGE, ROUTING_KEY, client.amqpProperties, url.toString().getBytes("UTF8"));
+ }
+}
71 src/test/java/com/simple/sftpfetch/publish/S3Test.java
@@ -0,0 +1,71 @@
+package com.simple.sftpfetch.publish;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3Client;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URL;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+public class S3Test {
+ public static final String BUCKET = "bucket";
+ private AmazonS3Client client;
+
+ @Before
+ public void setUp() throws Exception {
+ client = mock(AmazonS3Client.class);
+ when(client.doesBucketExist(BUCKET)).thenReturn(true);
+ }
+
+ @Test(expected=AmazonServiceException.class)
+ public void shouldVerifyThatTheBucketExists() {
+ when(client.doesBucketExist(BUCKET)).thenReturn(false);
+ new S3(client, BUCKET);
+ }
+
+ @Test
+ public void keyExistsShouldReturnFalseOn404() {
+ S3 s3 = new S3(client, BUCKET);
+ String key = "key.that.does.not.exist";
+
+ AmazonServiceException exception = new AmazonServiceException("Not found yo!");
+ exception.setStatusCode(404);
+ when(client.getObjectMetadata(BUCKET, key)).thenThrow(exception);
+ assertFalse("The key should not exist", s3.keyExists(key));
+ }
+
+ @Test
+ public void shouldUploadToTheCorrectBucket() {
+ S3 s3 = new S3(client, BUCKET);
+ String key = "the.key";
+ File toUpload = new File("/foo/bar/baz");
+ s3.upload(key, toUpload);
+ verify(client).putObject(BUCKET, key, toUpload);
+ }
+
+ @Test
+ public void shouldConstructCorrectUrlForUSStandard() throws Exception {
+ when(client.getBucketLocation(BUCKET)).thenReturn("US");
+ S3 s3 = new S3(client, BUCKET);
+
+ URL url = s3.getURLFor("formal.jpg");
+ assertEquals("https://s3.amazonaws.com/bucket/formal.jpg", url.toString());
+ }
+
+ @Test
+ public void shouldConstructCorrectUrlOutsideUSStandard() throws Exception {
+ when(client.getBucketLocation(BUCKET)).thenReturn("us-west-2");
+ S3 s3 = new S3(client, BUCKET);
+
+ URL url = s3.getURLFor("formal.jpg");
+ assertEquals("https://s3-us-west-2.amazonaws.com/bucket/formal.jpg", url.toString());
+ }
+}
88 src/test/java/com/simple/sftpfetch/sftp/SftpClientTest.java
@@ -0,0 +1,88 @@
+package com.simple.sftpfetch.sftp;
+
+import com.jcraft.jsch.*;
+import org.joda.time.DateTime;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Vector;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
+
+public class SftpClientTest {
+
+ public static final String USERNAME = "username";
+ public static final String PASSWORD = "password";
+ public static final String HOSTNAME = "host";
+ public static final int PORT = 22;
+ public static final int TIMEOUT = 500;
+
+ private JSch jSch = mock(JSch.class);
+ private Session session = mock(Session.class);
+ private ChannelSftp sftp = mock(ChannelSftp.class);
+ private SftpConnectionInfo connectionInfo = new SftpConnectionInfo(USERNAME, PASSWORD, DOWNLOAD_FROM, HOSTNAME, PORT, TIMEOUT);
+ private static final String DOWNLOAD_FROM = "OUT";
+
+ @Before
+ public void setUp() throws JSchException {
+ when(jSch.getSession(eq(USERNAME), eq(HOSTNAME), eq(PORT))).thenReturn(session);
+ when(session.openChannel("sftp")).thenReturn(sftp);
+ }
+
+ @Test
+ public void connectingShouldEstablishSessionAndChannelUsingSuppliedInformation() throws Exception {
+ new SftpClient(jSch, connectionInfo);
+
+ verify(session).setUserInfo(eq(new PasswordBasedAuthentication(PASSWORD)));
+ verify(session).connect(TIMEOUT);
+ verify(sftp).connect(TIMEOUT);
+ }
+
+ @Test
+ public void shouldGetFilesNewerThanTheNumberOfSuppliedDays() throws Exception {
+ SftpClient client = new SftpClient(jSch, connectionInfo);
+
+ ChannelSftp.LsEntry older = lsEntryWithGivenFilenameAndMTime("old.file", unixTimestampForDaysAgo(30));
+ ChannelSftp.LsEntry newer = lsEntryWithGivenFilenameAndMTime("new.file", unixTimestampForDaysAgo(2));
+
+ when(sftp.ls(DOWNLOAD_FROM)).thenReturn(new Vector<Object>(asList(older, newer)));
+ Set<String> files = client.getFilesNewerThan(7);
+
+ assertContainsOnly(files, new HashSet<String>(asList("new.file")));
+ }
+
+ @Test
+ public void closeShouldDisconnectChannelAndSession() throws Exception {
+ SftpClient client = new SftpClient(jSch, connectionInfo);
+ client.close();
+
+ verify(sftp).disconnect();
+ verify(session).disconnect();
+ }
+
+ private long unixTimestampForDaysAgo(int days) {
+ return new DateTime().minusDays(days).getMillis() / 1000;
+ }
+
+ private ChannelSftp.LsEntry lsEntryWithGivenFilenameAndMTime(String filename, long mtime) {
+ ChannelSftp.LsEntry lsEntry = mock(ChannelSftp.LsEntry.class);
+ SftpATTRS attrs = mock(SftpATTRS.class);
+ when(lsEntry.getAttrs()).thenReturn(attrs);
+ when(lsEntry.getFilename()).thenReturn(filename);
+ when(attrs.getMTime()).thenReturn((int) mtime);
+ return lsEntry;
+ }
+
+ public void assertContainsOnly(Set<String> collection, Set<String> elementsToAssert) {
+ assertTrue("Did not find all expected elements", collection.containsAll(elementsToAssert));
+ assertEquals("Found more elements than expected", collection.size(), elementsToAssert.size());
+ }
+}

0 comments on commit 8d0a03a

Please sign in to comment.
Something went wrong with that request. Please try again.