diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..26507f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.classpath +.project +.settings/ +.recommenders/ +target/ +.metadata +Servers/ +RemoteSystemTempFiles/ +fastDataTransfer/ \ No newline at end of file diff --git a/lib/globus/bcprov-jdk15on-1.50.jar b/lib/globus/bcprov-jdk15on-1.50.jar deleted file mode 100644 index d4b510d..0000000 Binary files a/lib/globus/bcprov-jdk15on-1.50.jar and /dev/null differ diff --git a/lib/globus/bouncycastle.LICENSE b/lib/globus/bouncycastle.LICENSE deleted file mode 100644 index 942c10c..0000000 --- a/lib/globus/bouncycastle.LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -Copyright (c) 2000 The Legion Of The Bouncy Castle (http://www.bouncycastle.org) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, -modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the -Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE -WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/pom.xml b/pom.xml index d32e103..41e59ae 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ jar - 1.7 + 1.8 UTF-8 @@ -16,41 +16,57 @@ edu.illinois.ncsa BCGSS 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/BCGSS.jar + org.bouncycastle - bcprov - 1.50.0 + bcprov-jdk15on + 1.50 + org.apache.commons commons-compress - 1.2.0 + 1.2 net.iharder.dnd filedrop 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/filedrop.jar + - com.sshtools.common + sshtools j2ssh-common 0.2.7 + system + ${project.basedir}/lib/gsi-sshterm/j2ssh-common-0.2.7.jar + - com.sshtools.core + sshtools j2ssh-core 0.2.7 + system + ${project.basedir}/lib/gsi-sshterm/j2ssh-core-0.2.7.jar org.lirc.socket jlirc-unix-soc 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/jlirc-unix-soc.jar uk.ac.rl.esc.browser libbrowser 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/libbrowser.jar log4j @@ -61,9 +77,12 @@ edu.illinois.ncsa ncsa-lcrypto 1.4.6 + system + ${project.basedir}/lib/gsi-sshterm/ncsa-lcrypto-146.jar + - org.apache.commons + ca.juliusdavies not-yet-commons-ssl 0.3.11 @@ -71,46 +90,64 @@ com.sshtools.ext putty-pk 1.1.0 + system + ${project.basedir}/lib/gsi-sshterm/putty-pk-1.1.0.jar com.sshtools.tunnel SecureTunneling 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/SecureTunneling.jar com.sshtools.shift ShiFT 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/ShiFT.jar com.sshtools.sshterm SSHTerm 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/SSHTerm-1.0.0.jar com.sshtools.sshvnc SSHVnc 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/SSHVnc.jar org.jdesktop swing-layout 1.0.3 + system + ${project.basedir}/lib/gsi-sshterm/swing-layout-1.0.3.jar org.globusonline TransferAPIClient 1.0.0 + system + ${project.basedir}/lib/gsi-sshterm/TransferAPIClient.jar org.italiangrid voms-api-java 3.0.1 + system + ${project.basedir}/lib/gsi-sshterm/voms-api-java-2.0.9.jar org.globus axisg 2.2.0 + system + ${project.basedir}/lib/gsi-sshterm/jglobus2/axisg-2.2.0.jar commons-logging @@ -126,21 +163,29 @@ org.globus gram 2.2.0 + system + ${project.basedir}/lib/globus/gram-2.2.0.jar org.globus gridftp 2.2.0 + system + ${project.basedir}/lib/globus/gridftp-2.2.0.jar org.globus.gsi.gssapi gss 2.2.0 + system + ${project.basedir}/lib/globus/gss-2.2.0.jar org.globus.io io 2.2.0 + system + ${project.basedir}/lib/globus/io-2.2.0.jar org.apache.httpcomponents @@ -156,6 +201,8 @@ org.globus.jsse jsse 2.2.0 + system + ${project.basedir}/lib/globus/jsse-2.2.0.jar com.google.guava @@ -171,16 +218,22 @@ org.globus.myproxy myproxy 2.2.0 + system + ${project.basedir}/lib/globus/myproxy-2.2.0.jar org.globus ssl-proxies 2.2.0 + system + ${project.basedir}/lib/globus/ssl-proxies-2.2.0.jar org.opentsdb opentsdb-client 2.1.0 + system + ${project.basedir}/lib/opentsdb/opentsdb-client-2.1.0.jar @@ -262,7 +315,8 @@ - java -jar fdt.jar "$@" + java + -jar fdt.jar "$@" 755 diff --git a/project b/project deleted file mode 100644 index efb71dc..0000000 --- a/project +++ /dev/null @@ -1,23 +0,0 @@ - - - FDT - - - - - - org.eclipse.jdt.core.javabuilder - - - - - edu.umd.cs.findbugs.plugin.eclipse.findbugsBuilder - - - - - - org.eclipse.jdt.core.javanature - edu.umd.cs.findbugs.plugin.eclipse.findbugsNature - - diff --git a/src/lia/util/net/common/Config.java b/src/lia/util/net/common/Config.java index 987be8f..8493bf1 100644 --- a/src/lia/util/net/common/Config.java +++ b/src/lia/util/net/common/Config.java @@ -7,6 +7,7 @@ import lia.util.net.copy.PosixFSFileChannelProviderFactory; import org.opentsdb.client.HttpClientImpl; import java.io.BufferedReader; +import java.io.File; import java.io.FileReader; import java.io.IOException; import java.net.*; @@ -300,6 +301,8 @@ public class Config { portNo = Utils.getIntValue(configMap, "-p", DEFAULT_PORT_NO); transportPorts = Utils.getTransportPortsValue(configMap, "-tp", DEFAULT_TRANSFER_PORT_NO); + if(transportPorts.size() == 1 && transportPorts.element().intValue() == DEFAULT_TRANSFER_PORT_NO) + transportPorts.clear(); tp = Arrays.asList(transportPorts.toArray()); isCoordinatorMode = (configMap.get("-coord") != null); isThirdPartyCopyAgent = (configMap.get("-agent") != null); @@ -703,7 +706,7 @@ public class Config { } private String[] getLogFiles(String sessionID) { - return new String[]{"/tmp/" + sessionID + ".log"}; + return new String[] { System.getProperty("java.io.tmpdir") + File.pathSeparatorChar + sessionID + ".log" }; } public String getListFilesFrom() { @@ -905,12 +908,16 @@ public class Config { } public int getNewRemoteTransferPort() { + int rtp = -1; try { if (!transportPorts.isEmpty()) { - int rtp = this.transportPorts.poll(20, TimeUnit.SECONDS); - System.out.println("Took new remote transfer port " + rtp); - return rtp; + rtp = this.transportPorts.poll(20, TimeUnit.SECONDS); + logger.log(Level.FINER,"Reusing remote transfer port " + rtp); + } else { + rtp = findAvailablePort(); + logger.log(Level.FINER,"Used new remote transfer port " + rtp); } + } catch (Exception e) { if (transportPorts.size() == 0) { logger.log(Level.WARNING, "No transfer ports defined or no free transfer ports left...", e); @@ -918,10 +925,31 @@ public class Config { logger.log(Level.WARNING, "Failed to retrieve remote transfer port", e); } } - return -1; - } - - public void setSessionSocket(ServerSocketChannel ssc, ServerSocket ss, SocketChannel sc, Socket s, int port) { + + return rtp; + } + + private int findAvailablePort() { + + /** + * Returns a free port number on localhost. + * @since December 2017 + * @author will + * @return a free port number on localhost + * @throws IllegalStateException if unable to find a free port + */ + try(ServerSocket socket = new ServerSocket(0)) { + socket.setReuseAddress(true); + return socket.getLocalPort(); + } catch(IOException e) + { + logger.log(Level.WARNING, "Unable to find a free Socket", e); + } + + throw new IllegalStateException("Could not find a free TCP/IP port"); + } + + public void setSessionSocket(ServerSocketChannel ssc, ServerSocket ss, SocketChannel sc, Socket s, int port) { List socks = new ArrayList<>(); socks.add(ssc); socks.add(ss); diff --git a/src/lia/util/net/common/Utils.java b/src/lia/util/net/common/Utils.java index 58a12ff..13f5db5 100644 --- a/src/lia/util/net/common/Utils.java +++ b/src/lia/util/net/common/Utils.java @@ -1557,7 +1557,6 @@ public final class Utils { } } - @SuppressWarnings("SameParameterValue") public static void cancelFutureIgnoringException(Future f, boolean mayInterruptIfRunning) { if (f != null) { try { diff --git a/src/lia/util/net/copy/FDTReaderSession.java b/src/lia/util/net/copy/FDTReaderSession.java index aaaa38f..a6fbf88 100644 --- a/src/lia/util/net/copy/FDTReaderSession.java +++ b/src/lia/util/net/copy/FDTReaderSession.java @@ -60,7 +60,7 @@ public class FDTReaderSession extends FDTSession implements FileBlockProducer { */ public FDTReaderSession(int transferPort) throws Exception { super(FDTSession.CLIENT, transferPort); - Utils.initLogger(config.getLogLevel(), new File("/tmp/" + sessionID + ".log"), new Properties()); + Utils.initLogger(config.getLogLevel(), new File(System.getProperty("java.io.tmpdir") + File.pathSeparatorChar + sessionID + ".log"), new Properties()); final int rMul = Integer.getInteger("fdt.rQueueM", 2); final int avProcProp = Integer.getInteger("fdt.avProc", 1); final int avProcMax = Math.max(avProcProp, Utils.availableProcessors()); @@ -88,7 +88,7 @@ public class FDTReaderSession extends FDTSession implements FileBlockProducer { */ public FDTReaderSession(ControlChannel ctrlChannel) throws Exception { super(ctrlChannel, FDTSession.SERVER); - Utils.initLogger(config.getLogLevel(), new File("/tmp/" + sessionID + ".log"), new Properties()); + Utils.initLogger(config.getLogLevel(), new File(System.getProperty("java.io.tmpdir") + File.pathSeparatorChar + sessionID + ".log"), new Properties()); fileBlockQueue = new ArrayBlockingQueue(Utils.availableProcessors() * 2); readersMap = new TreeMap>(); diff --git a/src/lia/util/net/copy/FDTWriterSession.java b/src/lia/util/net/copy/FDTWriterSession.java index f682771..823a1a9 100644 --- a/src/lia/util/net/copy/FDTWriterSession.java +++ b/src/lia/util/net/copy/FDTWriterSession.java @@ -15,6 +15,8 @@ import java.io.File; import java.net.InetAddress; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,7 +48,7 @@ public class FDTWriterSession extends FDTSession implements FileBlockConsumer { public FDTWriterSession(int transferPort) throws Exception { super(FDTSession.CLIENT, transferPort); - Utils.initLogger(config.getLogLevel(), new File("/tmp/" + sessionID + ".log"), new Properties()); + Utils.initLogger(config.getLogLevel(), new File(System.getProperty("java.io.tmpdir") + File.pathSeparatorChar + sessionID + ".log"), new Properties()); dwm.addSession(this); sendInitConf(); this.monID = config.getMonID(); @@ -424,9 +426,13 @@ public class FDTWriterSession extends FDTSession implements FileBlockConsumer { for (int i = 0; i < fCount; i++) { final String fName = (sccm.remappedFileLists == null || sccm.remappedFileLists[i] == null) ? sccm.fileLists[i] : sccm.remappedFileLists[i]; + // OSG change, just get the file name and none of the path stuff + Path p = Paths.get(fName); + String oFileName = p.getFileName().toString(); + final FileWriterSession fws = new FileWriterSession(sccm.fileIDs[i], this, this.destinationDir + File.separator - + ((shouldReplace) ? fName.replace(remoteCharSeparator, File.separatorChar) : fName), + + ((shouldReplace) ? fName.replace(remoteCharSeparator, File.separatorChar) : oFileName), sccm.fileSizes[i], sccm.lastModifTimes[i], isLoop, writeMode, noTmp, noLock, fcp); fileSessions.put(fws.sessionID, fws); if (hasPreProc) { diff --git a/src/lia/util/net/copy/transport/ControlChannel.java b/src/lia/util/net/copy/transport/ControlChannel.java index a1dd59f..449e28d 100644 --- a/src/lia/util/net/copy/transport/ControlChannel.java +++ b/src/lia/util/net/copy/transport/ControlChannel.java @@ -280,7 +280,17 @@ public class ControlChannel extends AbstractFDTCloseable implements Runnable { if (cleanupFinished.compareAndSet(false, true)) { Utils.cancelFutureIgnoringException(ccptFuture, false); Utils.closeIgnoringExceptions(ois); + if(oos != null) + try { + oos.flush(); + } catch (IOException e) { + } Utils.closeIgnoringExceptions(oos); + // give the output stream time to flush + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } Utils.closeIgnoringExceptions(controlSocket); if (notifier != null) { @@ -418,8 +428,8 @@ public class ControlChannel extends AbstractFDTCloseable implements Runnable { logger.log(Level.FINEST, "[ ControlChannel ] [ getResponse] waited for " + RETRY_TIMEOUT + "ms"); } finally { - if (newCtrlMsg == null && i == MAX_RETRIES) { - logger.log(Level.FINEST, "[ ControlChannel ] [ getResponse] CtrlMsg " + newCtrlMsg, t); + if (newCtrlMsg == null && i >= MAX_RETRIES) { + logger.log(Level.FINEST, "[ ControlChannel ] [ getResponse] CtrlMsg " + newCtrlMsg , t); throw t; } } @@ -542,6 +552,7 @@ public class ControlChannel extends AbstractFDTCloseable implements Runnable { notifQueue.add(o); } catch (SocketTimeoutException ste) { + logger.log(Level.FINEST, "Control Channel timeout"); // ignore this??? or shall I close it() ? } catch (IOException ioe) { close("Control channel got I/O Exception", ioe); @@ -555,7 +566,6 @@ public class ControlChannel extends AbstractFDTCloseable implements Runnable { } catch (Throwable t) { if (!isClosed()) { - internalDownMsg = myName + " got exception in main loop: " + t.getMessage(); internalDownCause = t; @@ -570,6 +580,7 @@ public class ControlChannel extends AbstractFDTCloseable implements Runnable { } else { close(internalDownMsg, internalDownCause); } + cleanup(); } logger.log(Level.INFO, myName + " FINISHED");