Merge remote-tracking branch 'remotes/origin/dev'

project.clj

@@ -24,7 +24,8 @@
                  [clojurewerkz/elastisch "1.0.2"]
                  [hoot "0.1.0-SNAPSHOT"]
                  [com.novemberain/validateur "1.4.0"]
-                 [xerces/xercesImpl "2.11.0"]]
+                 [xerces/xercesImpl "2.11.0"]
+                 [commons-net "3.3"]]
   :plugins [[org.iplantc/lein-iplant-rpm "1.4.3-SNAPSHOT"]
             [lein-ring "0.7.4"]
             [swank-clojure "1.4.2"]]

src/donkey/services/fileio/controllers.clj

@@ -1,5 +1,5 @@
 (ns donkey.services.fileio.controllers
-  (:use [clj-jargon.jargon] 
+  (:use [clj-jargon.jargon]
         [clojure-commons.error-codes]
         [donkey.util.config]
         [donkey.util.validators]
@@ -20,12 +20,10 @@
   [address]
   (try+
    (ssl/input-stream address)
-   (catch java.net.UnknownHostException e
+   (catch java.io.IOException e
      (throw+ {:error_code ERR_INVALID_URL
-              :url address}))
-   (catch java.net.MalformedURLException e
-     (throw+ {:error_code ERR_INVALID_URL
-              :url address}))))
+              :url address
+              :msg (.getMessage e)}))))
 
 (defn gen-uuid []
   (str (java.util.UUID/randomUUID)))
@@ -62,7 +60,7 @@
         up-path (get req-multipart "file")]
     (if-not (fs/good-string? up-path)
       {:status 500
-       :body   (json/generate-string 
+       :body   (json/generate-string
                  {:error_code ERR_BAD_OR_MISSING_FIELD
                   :path       up-path})}
       (actions/upload user up-path dest))))
@@ -91,12 +89,13 @@
     (let [user    (:user params)
           dest    (string/trim (:dest body))
           addr    (string/trim (:address body))
-          istream (in-stream addr)
           fname   (url-filename addr)]
       (log/warn (str "User: " user))
       (log/warn (str "Dest: " dest))
       (log/warn (str "Fname: " fname))
       (log/warn (str "Addr: " addr))
+      (with-open [istream (in-stream addr)]
+        (log/warn "connection to" addr "successfully established"))
       (actions/urlimport user addr fname dest))))
 
 (defn saveas
@@ -112,23 +111,23 @@
         (when-not (user-exists? cm user)
           (throw+ {:user       user
                    :error_code ERR_NOT_A_USER}))
-        
+
         (when-not (exists? cm (ft/dirname dest))
           (throw+ {:error_code ERR_DOES_NOT_EXIST
                    :path       (ft/dirname dest)}))
-        
+
         (when-not (is-writeable? cm user (ft/dirname dest))
           (throw+ {:error_code ERR_NOT_WRITEABLE
                    :path       (ft/dirname dest)}))
-        
+
         (when (exists? cm dest)
-          (throw+ {:error_code ERR_EXISTS 
+          (throw+ {:error_code ERR_EXISTS
                    :path       dest}))
-        
+
         (with-in-str cont
           (actions/store cm *in* user dest)
           {:status "success"
-           :file 
+           :file
            {:id dest
             :label         (ft/basename dest)
             :permissions   (dataobject-perm-map cm user dest)

src/donkey/util/ssl.clj

@@ -1,6 +1,8 @@
 (ns donkey.util.ssl
   [:import [java.net URL]]
-  [:import [java.io IOException]]
+  [:import [java.io
+            InputStream
+            IOException]]
   [:import [javax.net.ssl
             HostnameVerifier
             HttpsURLConnection
@@ -14,7 +16,10 @@
             SecureRandom]]
   [:import [java.security.cert
             CertificateException
-            X509Certificate]])
+            X509Certificate]]
+  [:import [org.apache.commons.net.ftp
+            FTPClient
+            FTPReply]])
 
 (def trust-manager
   (proxy [X509TrustManager] []
@@ -43,14 +48,65 @@
       (do
         (HttpsURLConnection/setDefaultSSLSocketFactory (. ssl-context getSocketFactory))
         (HttpsURLConnection/setDefaultHostnameVerifier hostname-verifier)
-        (. url openConnection))
+        (.openConnection url))
       (catch GeneralSecurityException e
         (throw IOException "Unable to establish trusting SSL connection." e))
       (finally
         (do
           (HttpsURLConnection/setDefaultSSLSocketFactory orig-socket-factory)
           (HttpsURLConnection/setDefaultHostnameVerifier orig-hostname-verifier))))))
 
+(defn- ftp-connect
+  [ftp url]
+  (if (pos? (.getPort url))
+    (.connect ftp (.getHost url) (.getPort url))
+    (.connect ftp (.getHost url))))
+
+(defn- ftp-login
+  ([ftp username password]
+     (.login ftp username password))
+  ([ftp url]
+     (if-let [user-info (.getUserInfo url)]
+       (apply ftp-login (clojure.string/split #":" user-info))
+       (ftp-login ftp "anonymous" ""))))
+
+(defn- get-ftp-input-stream
+  ([ftp url]
+     (ftp-connect ftp url)
+     (.enterLocalPassiveMode ftp)
+     (when-not (FTPReply/isPositiveCompletion (.getReplyCode ftp))
+       (throw (IOException. "FTP server refused connection")))
+     (when-not (ftp-login ftp url)
+       (throw (IOException. "FTP server rejected credentials")))
+     (.retrieveFileStream ftp (.getPath url)))
+  ([url]
+     (let [ftp (FTPClient.)
+           in  (get-ftp-input-stream ftp url)]
+       (proxy [InputStream] []
+         (available []
+           (.available in))
+         (close []
+           (.close in)
+           (.logout ftp)
+           (.disconnect ftp))
+         (mark [read-limit]
+           (.mark in read-limit))
+         (markSupported []
+           (.markSupported in))
+         (read
+           ([]
+              (.read in))
+           ([b]
+              (.read in b))
+           ([b off len]
+              (.read in b off len)))
+         (reset []
+           (.reset in))
+         (skip [n]
+           (.skip in n))))))
+
 (defn input-stream [url-string]
   (let [url (URL. url-string)]
-    (. (get-connection url) getInputStream)))
+    (if (= (.getProtocol url) "ftp")
+      (get-ftp-input-stream url)
+      (.getInputStream (get-connection url)))))