Skip to content

Commit

Permalink
[PDI-14179] - FTP put error when using ${Internal.Job.Filename.Direct…
Browse files Browse the repository at this point in the history 
…ory} internal variable

- use URI to handle paths with file:// prefix
- fix Checkstyle violations
  • Loading branch information
Andrey Khayrutdinov committed Oct 23, 2015
1 parent c2964f7 commit f400847
Showing 1 changed file with 60 additions and 86 deletions.
146 changes: 60 additions & 86 deletions engine/src/org/pentaho/di/job/entries/ftpput/JobEntryFTPPUT.java
View file
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2013 by Pentaho : http://www.pentaho.com
* Copyright (C) 2002-2015 by Pentaho : http://www.pentaho.com
*
*******************************************************************************
*
Expand Down Expand Up @@ -32,7 +32,9 @@
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand Down Expand Up @@ -73,7 +75,6 @@
*
* @author Samatar
* @since 15-09-2007
*
*/

public class JobEntryFTPPUT extends JobEntryBase implements Cloneable, JobEntryInterface {
Expand Down Expand Up @@ -108,12 +109,12 @@ public class JobEntryFTPPUT extends JobEntryBase implements Cloneable, JobEntryI
/**
* Implicit encoding used before PDI v2.4.1
*/
private static String LEGACY_CONTROL_ENCODING = "US-ASCII";
private static final String LEGACY_CONTROL_ENCODING = "US-ASCII";

/**
* Default encoding when making a new ftp job entry instance.
*/
private static String DEFAULT_CONTROL_ENCODING = "ISO-8859-1";
private static final String DEFAULT_CONTROL_ENCODING = "ISO-8859-1";

public JobEntryFTPPUT( String n ) {
super( n, "" );
Expand Down Expand Up @@ -170,7 +171,7 @@ public String getXML() {
}

public void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveServer> slaveServers,
Repository rep, IMetaStore metaStore ) throws KettleXMLException {
Repository rep, IMetaStore metaStore ) throws KettleXMLException {
try {
super.loadXML( entrynode, databases, slaveServers );
serverName = XMLHandler.getTagValue( entrynode, "servername" );
Expand Down Expand Up @@ -209,7 +210,7 @@ public void loadXML( Node entrynode, List<DatabaseMeta> databases, List<SlaveSer
}

public void loadRep( Repository rep, IMetaStore metaStore, ObjectId id_jobentry, List<DatabaseMeta> databases,
List<SlaveServer> slaveServers ) throws KettleException {
List<SlaveServer> slaveServers ) throws KettleException {
try {
serverName = rep.getJobEntryAttributeString( id_jobentry, "servername" );
serverPort = rep.getJobEntryAttributeString( id_jobentry, "serverport" );
Expand Down Expand Up @@ -292,16 +293,14 @@ public boolean isBinaryMode() {
}

/**
* @param binaryMode
* The binaryMode to set.
* @param binaryMode The binaryMode to set.
*/
public void setBinaryMode( boolean binaryMode ) {
this.binaryMode = binaryMode;
}

/**
* @param timeout
* The timeout to set.
* @param timeout The timeout to set.
*/
public void setTimeout( int timeout ) {
this.timeout = timeout;
Expand All @@ -322,8 +321,7 @@ public boolean isOnlyPuttingNewFiles() {
}

/**
* @param onlyPuttingNewFiles
* Only transfer new files to the remote host
* @param onlyPuttingNewFiles Only transfer new files to the remote host
*/
public void setOnlyPuttingNewFiles( boolean onlyPuttingNewFiles ) {
this.onlyPuttingNewFiles = onlyPuttingNewFiles;
Expand All @@ -342,8 +340,7 @@ public String getControlEncoding() {
* Set the encoding to be used for ftp'ing. This determines how names are translated in dir e.g. It does impact the
* contents of the files being ftp'ed.
*
* @param encoding
* The encoding to be used.
* @param encoding The encoding to be used.
*/
public void setControlEncoding( String encoding ) {
this.controlEncoding = encoding;
Expand All @@ -357,8 +354,7 @@ public String getRemoteDirectory() {
}

/**
* @param directory
* The remoteDirectory to set.
* @param directory The remoteDirectory to set.
*/
public void setRemoteDirectory( String directory ) {
this.remoteDirectory = directory;
Expand All @@ -372,8 +368,7 @@ public String getPassword() {
}

/**
* @param password
* The password to set.
* @param password The password to set.
*/
public void setPassword( String password ) {
this.password = password;
Expand All @@ -387,8 +382,7 @@ public String getServerName() {
}

/**
* @param serverName
* The serverName to set.
* @param serverName The serverName to set.
*/
public void setServerName( String serverName ) {
this.serverName = serverName;
Expand All @@ -402,8 +396,7 @@ public String getUserName() {
}

/**
* @param userName
* The userName to set.
* @param userName The userName to set.
*/
public void setUserName( String userName ) {
this.userName = userName;
Expand All @@ -417,8 +410,7 @@ public String getWildcard() {
}

/**
* @param wildcard
* The wildcard to set.
* @param wildcard The wildcard to set.
*/
public void setWildcard( String wildcard ) {
this.wildcard = wildcard;
Expand All @@ -432,16 +424,14 @@ public String getLocalDirectory() {
}

/**
* @param directory
* The localDirectory to set.
* @param directory The localDirectory to set.
*/
public void setLocalDirectory( String directory ) {
this.localDirectory = directory;
}

/**
* @param remove
* The remove to set.
* @param remove The remove to set.
*/
public void setRemove( boolean remove ) {
this.remove = remove;
Expand Down Expand Up @@ -470,8 +460,7 @@ public boolean isActiveConnection() {
}

/**
* @param activeConnection
* set to true to get an active FTP connection
* @param activeConnection set to true to get an active FTP connection
*/
public void setActiveConnection( boolean activeConnection ) {
this.activeConnection = activeConnection;
Expand All @@ -485,8 +474,7 @@ public String getProxyHost() {
}

/**
* @param proxyHost
* The hostname of the proxy.
* @param proxyHost The hostname of the proxy.
*/
public void setProxyHost( String proxyHost ) {
this.proxyHost = proxyHost;
Expand All @@ -500,8 +488,7 @@ public String getProxyPassword() {
}

/**
* @param proxyPassword
* The password which is used to authenticate at the proxy.
* @param proxyPassword The password which is used to authenticate at the proxy.
*/
public void setProxyPassword( String proxyPassword ) {
this.proxyPassword = proxyPassword;
Expand All @@ -515,8 +502,7 @@ public String getProxyPort() {
}

/**
* @param proxyPort
* The port of the ftp-proxy.
* @param proxyPort The port of the ftp-proxy.
*/
public void setProxyPort( String proxyPort ) {
this.proxyPort = proxyPort;
Expand All @@ -530,35 +516,28 @@ public String getProxyUsername() {
}

/**
* @param socksProxyHost
* The socks proxy host to set
* @param socksProxyHost The socks proxy host to set
*/
public void setSocksProxyHost( String socksProxyHost ) {
this.socksProxyHost = socksProxyHost;
}

/**
*
* @param socksProxyPort
* The socks proxy port to set
* @param socksProxyPort The socks proxy port to set
*/
public void setSocksProxyPort( String socksProxyPort ) {
this.socksProxyPort = socksProxyPort;
}

/**
*
* @param socksProxyUsername
* The socks proxy username to set
* @param socksProxyUsername The socks proxy username to set
*/
public void setSocksProxyUsername( String socksProxyUsername ) {
this.socksProxyUsername = socksProxyUsername;
}

/**
*
* @param socksProxyPassword
* The socks proxy password to set
* @param socksProxyPassword The socks proxy password to set
*/
public void setSocksProxyPassword( String socksProxyPassword ) {
this.socksProxyPassword = socksProxyPassword;
Expand Down Expand Up @@ -593,8 +572,7 @@ public String getSocksProxyPassword() {
}

/**
* @param proxyUsername
* The username which is used to authenticate at the proxy.
* @param proxyUsername The username which is used to authenticate at the proxy.
*/
public void setProxyUsername( String proxyUsername ) {
this.proxyUsername = proxyUsername;
Expand Down Expand Up @@ -722,34 +700,29 @@ public Result execute( Result previousResult, int nr ) {
}
}

// Get all the files in the local directory...
int x = 0;

// Joerg: ..that's for Java5
// ArrayList<String> myFileList = new ArrayList<String>();
ArrayList<String> myFileList = new ArrayList<String>();
// handle file:/// prefix
if ( realLocalDirectory.startsWith( "file:" ) ) {
realLocalDirectory = new URI( realLocalDirectory ).getPath();
}

final List<String> files;
File localFiles = new File( realLocalDirectory );
File[] children = localFiles.listFiles();
for ( int i = 0; i < children.length; i++ ) {
// Get filename of file or directory
if ( !children[i].isDirectory() ) {
// myFileList.add(children[i].getAbsolutePath());
myFileList.add( children[i].getName() );
x = x + 1;

if ( children == null ) {
files = Collections.emptyList();
} else {
files = new ArrayList<String>( children.length );
for ( File child : children ) {
// Get filename of file or directory
if ( !child.isDirectory() ) {
files.add( child.getName() );
}
}
} // end for

// Joerg: ..that's for Java5
// String[] filelist = myFileList.toArray(new String[myFileList.size()]);

String[] filelist = new String[myFileList.size()];
myFileList.toArray( filelist );
}

if ( log.isDetailed() ) {
logDetailed( BaseMessages.getString(
PKG, "JobFTPPUT.Log.FoundFileLocalDirectory", "" + filelist.length, realLocalDirectory ) );
PKG, "JobFTPPUT.Log.FoundFileLocalDirectory", "" + files.size(), realLocalDirectory ) );
}

Pattern pattern = null;
Expand All @@ -758,43 +731,44 @@ public Result execute( Result previousResult, int nr ) {

} // end if

// Get the files in the list and execute ftp.put() for each file
for ( int i = 0; i < filelist.length && !parentJob.isStopped(); i++ ) {
boolean getIt = true;
for ( String file : files ) {
if ( parentJob.isStopped() ) {
break;
}

boolean toBeProcessed = true;

// First see if the file matches the regular expression!
if ( pattern != null ) {
Matcher matcher = pattern.matcher( filelist[i] );
getIt = matcher.matches();
Matcher matcher = pattern.matcher( file );
toBeProcessed = matcher.matches();
}

if ( getIt ) {

if ( toBeProcessed ) {
// File exists?
boolean fileExist = false;
try {
fileExist = ftpclient.exists( filelist[i] );

fileExist = ftpclient.exists( file );
} catch ( Exception e ) {
// Assume file does not exist !!
}

if ( log.isDebug() ) {
if ( fileExist ) {
logDebug( BaseMessages.getString( PKG, "JobFTPPUT.Log.FileExists", filelist[i] ) );
logDebug( BaseMessages.getString( PKG, "JobFTPPUT.Log.FileExists", file ) );
} else {
logDebug( BaseMessages.getString( PKG, "JobFTPPUT.Log.FileDoesNotExists", filelist[i] ) );
logDebug( BaseMessages.getString( PKG, "JobFTPPUT.Log.FileDoesNotExists", file ) );
}
}

if ( !fileExist || ( !onlyPuttingNewFiles && fileExist ) ) {
if ( log.isDebug() ) {
logDebug( BaseMessages.getString(
PKG, "JobFTPPUT.Log.PuttingFileToRemoteDirectory", filelist[i], realRemoteDirectory ) );
PKG, "JobFTPPUT.Log.PuttingFileToRemoteDirectory", file, realRemoteDirectory ) );
}

String localFilename = realLocalDirectory + Const.FILE_SEPARATOR + filelist[i];
ftpclient.put( localFilename, filelist[i] );
String localFilename = realLocalDirectory + Const.FILE_SEPARATOR + file;
ftpclient.put( localFilename, file );

filesput++;

Expand Down Expand Up @@ -849,7 +823,7 @@ public List<ResourceReference> getResourceDependencies( JobMeta jobMeta ) {

@Override
public void check( List<CheckResultInterface> remarks, JobMeta jobMeta, VariableSpace space,
Repository repository, IMetaStore metaStore ) {
Repository repository, IMetaStore metaStore ) {
andValidator().validate( this, "serverName", remarks, putValidators( notBlankValidator() ) );
andValidator().validate(
this, "localDirectory", remarks, putValidators( notBlankValidator(), fileExistsValidator() ) );
Expand Down Expand Up @@ -898,7 +872,7 @@ protected void hookInOtherParsers( FTPClient ftpClient ) throws FTPException, IO
Class<?> clazz = null;
Object parserInstance = null;
for ( int i = 0; i < parserClasses.length; i++ ) {
cName = parserClasses[i].trim();
cName = parserClasses[ i ].trim();
if ( cName.length() > 0 ) {
try {
clazz = Class.forName( cName );
Expand Down

0 comments on commit f400847

Please sign in to comment.