Merge remote-tracking branch 'remotes/origin/dev'

conf/main/log4j.properties

@@ -1,4 +1,4 @@
-log4j.rootLogger=WARN, A
+log4j.rootLogger=WARN, A, JARGON
 
 # Uncomment these lines to enable debugging in Donkey itself.
 # log4j.category.donkey=DEBUG, A
@@ -13,6 +13,9 @@ log4j.rootLogger=WARN, A
 log4j.logger.JsonLogger=debug, JSON
 log4j.additivity.JsonLogger=false
 
+log4j.logger.org.irods.jargon=debug, JARGON
+log4j.additivity.org.irods.jargon=false
+
 # Use this appender for logging JSON when running Donkey in the background.
 log4j.appender.JSON=org.apache.log4j.RollingFileAppender
 log4j.appender.JSON.File=/var/log/donkey/json.log
@@ -28,3 +31,9 @@ log4j.appender.A.layout=org.apache.log4j.PatternLayout
 log4j.appender.A.layout.ConversionPattern=%d{MM-dd@HH:mm:ss} %-5p (%13F:%L) %3x - %m%n
 log4j.appender.A.MaxFileSize=10MB
 log4j.appender.A.MaxBackupIndex=1
+
+log4j.appender.JARGON=org.apache.log4j.RollingFileAppender
+log4j.appender.JARGON.File=/var/log/donkey/jargon.log
+log4j.appender.JARGON.MaxFileSize=100MB
+log4j.appender.JARGON.layout=org.apache.log4j.PatternLayout
+log4j.appender.JARGON.layout.ConversionPattern=%d{MM-dd@HH:mm:ss} %-5p (%13F:%L) %3x - %m%n

project.clj

@@ -1,6 +1,7 @@
 (defproject donkey "1.3.5-SNAPSHOT"
   :description "Framework for hosting DiscoveryEnvironment metadata services."
   :dependencies [[org.clojure/clojure "1.5.1"]
+                 [com.novemberain/langohr "1.4.0"]
                  [org.clojure/core.memoize "0.5.3"]
                  [org.clojure/tools.logging "0.2.3"]
                  [org.clojure/data.codec "0.1.0"]
@@ -11,6 +12,7 @@
                   :exclusions [[xerces/xmlParserAPIs]
                                [org.irods.jargon.transfer/jargon-transfer-dao-spring]]]
                  [org.iplantc/clojure-commons "1.4.5-SNAPSHOT"]
+                 [org.iplantc/mescal "0.1.0-SNAPSHOT"]
                  [org/forester "1.005" ]
                  [org.nexml.model/nexml "1.5-SNAPSHOT"]
                  [net.sf.json-lib/json-lib "2.4" :classifier "jdk15"]
@@ -25,15 +27,16 @@
                  [hoot "0.1.0-SNAPSHOT"]
                  [com.novemberain/validateur "1.4.0"]
                  [xerces/xercesImpl "2.11.0"]
-                 [commons-net "3.3"]]
+                 [commons-net "3.3"]
+                 [org.clojure/tools.nrepl "0.2.3"]]
   :plugins [[org.iplantc/lein-iplant-rpm "1.4.3-SNAPSHOT"]
             [lein-ring "0.7.4"]
             [swank-clojure "1.4.2"]]
   :profiles {:dev {:resource-paths ["conf/test"]}}
   :aot [donkey.core]
   :main donkey.core
   :ring {:handler donkey.core/app
-         :init donkey.core/load-configuration-from-file
+         :init donkey.core/lein-ring-init
          :port 31325}
   :iplant-rpm {:summary "iPlant Discovery Environment Business Layer Services"
                :provides "donkey"

src/donkey/clients/metadactyl.clj

@@ -34,3 +34,11 @@
                    :as           :stream})
       (:body)
       (service/decode-json)))
+
+(defn get-app
+  [app-id]
+  (-> (client/get (secured-url "app" app-id)
+                  {:query-params (secured-params)
+                   :as           :stream})
+      (:body)
+      (service/decode-json)))

src/donkey/core.clj

@@ -27,7 +27,8 @@
             [clojure.tools.logging :as log]
             [ring.adapter.jetty :as jetty]
             [donkey.util.config :as config]
-            [donkey.services.fileio.controllers :as fileio]))
+            [donkey.services.fileio.controllers :as fileio]
+            [clojure.tools.nrepl.server :as nrepl]))
 
 (defn- flagged-routes
   [& handlers]
@@ -69,11 +70,20 @@
 
    (route/not-found (unrecognized-path-response))))
 
+(defn start-nrepl
+  []
+  (nrepl/start-server :port 7888))
+
 (defn load-configuration-from-file
   "Loads the configuration properties from a file."
   []
   (config/load-config-from-file))
 
+(defn lein-ring-init
+  []
+  (load-configuration-from-file)
+  (start-nrepl))
+
 (defn load-configuration-from-zookeeper
   "Loads the configuration properties from Zookeeper."
   []
@@ -90,6 +100,7 @@
   (-> (delayed-handler donkey-routes)
     (wrap-multipart-params {:store fileio/store-irods})
     trap-handler
+    req-logger
     wrap-keyword-params
     wrap-lcase-params
     wrap-query-params))
@@ -108,4 +119,5 @@
   (load-configuration-from-zookeeper)
   (register-specific-queries)
   (log/warn "Listening on" (config/listen-port))
+  (start-nrepl)
   (jetty/run-jetty app {:port (config/listen-port)}))

src/donkey/routes/metadata.clj

@@ -5,7 +5,7 @@
         [donkey.util.service]
         [donkey.util])
   (:require [donkey.util.config :as config]
-            [donkey.services.metadata.app-listings :as app-listings]
+            [donkey.services.metadata.apps :as apps]
             [donkey.services.jex :as jex]))
 
 (defn secured-metadata-routes
@@ -22,8 +22,8 @@
    (GET "/template/:app-id" [app-id :as req]
         (trap #(get-app-secured req app-id)))
 
-   (GET "/app/:app-id" [app-id :as req]
-        (trap #(get-app-new-format req app-id)))
+   (GET "/app/:app-id" [app-id]
+        (trap #(apps/get-app app-id)))
 
    (PUT "/workspaces/:workspace-id/newexperiment" [workspace-id :as req]
         (trap #(run-experiment req workspace-id)))
@@ -50,13 +50,13 @@
         (trap #(search-apps req)))
 
    (GET "/app-groups" []
-        (trap #(app-listings/get-only-app-groups)))
+        (trap #(apps/get-only-app-groups)))
 
-   (GET "/get-analyses-in-group/:app-group-id" [app-group-id :as req]
-        (trap #(list-apps-in-group req app-group-id)))
+   (GET "/get-analyses-in-group/:app-group-id" [app-group-id]
+        (trap #(apps/apps-in-group app-group-id)))
 
-   (GET "/list-analyses-for-pipeline/:app-group-id" [app-group-id :as req]
-        (trap #(list-apps-in-group req app-group-id)))
+   (GET "/list-analyses-for-pipeline/:app-group-id" [app-group-id]
+        (trap #(apps/apps-in-group app-group-id)))
 
    (GET "/get-components-in-analysis/:app-id" [app-id :as req]
         (trap #(list-deployed-components-in-app req app-id)))

src/donkey/services/filesystem/controllers.clj

@@ -97,7 +97,7 @@
             home-f     (future (dir-list user (get-home-dir user) inc-files))]
         {:roots [@home-f @comm-f @share-f]})
 
-      (= (utils/add-trailing-slash (:path params)) (irods-home))
+      (= (utils/add-trailing-slash (:path params)) (utils/add-trailing-slash (irods-home)))
       (irods-actions/shared-root-listing
         (:user params)
         (irods-home)

src/donkey/services/metadata/apps.clj

@@ -0,0 +1,61 @@
+(ns donkey.services.metadata.apps
+  (:use [donkey.auth.user-attributes :only [current-user]])
+  (:require [donkey.clients.metadactyl :as metadactyl]
+            [donkey.util.config :as config]
+            [donkey.util.service :as service]
+            [mescal.de :as agave]))
+
+(def ^:private uuid-regex
+  #"^\p{XDigit}{8}(?:-\p{XDigit}{4}){3}-\p{XDigit}{12}$")
+
+(defprotocol AppLister
+  "Used to list apps available to the Discovery Environment."
+  (listAppGroups [this])
+  (listApps [this group-id])
+  (getApp [this app-id]))
+
+(deftype DeOnlyAppLister []
+  AppLister
+  (listAppGroups [this]
+    (metadactyl/get-only-app-groups))
+  (listApps [this group-id]
+    (metadactyl/apps-in-group group-id))
+  (getApp [this app-id]
+    (metadactyl/get-app app-id)))
+
+(deftype DeHpcAppLister [agave-client]
+  AppLister
+  (listAppGroups [this]
+    (-> (metadactyl/get-only-app-groups)
+        (update-in [:groups] conj (.publicAppGroup agave-client))))
+  (listApps [this group-id]
+    (if (= group-id (:id (.publicAppGroup agave-client)))
+      (.listPublicApps agave-client)
+      (metadactyl/apps-in-group group-id)))
+  (getApp [this app-id]
+    (if (re-find uuid-regex app-id)
+      (metadactyl/get-app app-id)
+      (.getApp agave-client app-id))))
+
+(defn- get-app-lister
+  []
+  (if (config/agave-enabled)
+    (DeHpcAppLister. (agave/de-agave-client-v1
+                      (config/agave-base-url)
+                      (config/agave-user)
+                      (config/agave-pass)
+                      (:shortUsername current-user)
+                      (config/agave-jobs-enabled)))
+    (DeOnlyAppLister.)))
+
+(defn get-only-app-groups
+  []
+  (service/success-response (.listAppGroups (get-app-lister))))
+
+(defn apps-in-group
+  [group-id]
+  (service/success-response (.listApps (get-app-lister) group-id)))
+
+(defn get-app
+  [app-id]
+  (service/success-response (.getApp (get-app-lister) app-id)))

src/donkey/services/metadata/metadactyl.clj

@@ -113,13 +113,6 @@
   (let [url (build-metadactyl-secured-url req "template" app-id)]
     (forward-get url req)))
 
-(defn get-app-new-format
-  "This service gets an app in the format required by the DE as of version 1.8."
-  [req app-id]
-  (forward-get
-   (build-metadactyl-secured-url req "app" app-id)
-   req))
-
 (defn get-public-app-groups
   "Retrieves the list of public app groups."
   [req]

src/donkey/util.clj

@@ -54,6 +54,12 @@
       (catch IllegalStateException e (failure-response e))
       (catch Throwable t (error-response t)))))
 
+(defn req-logger
+  [handler]
+  (fn [req]
+    (log/info "Request received:" req)
+    (handler req)))
+
 (defn as-vector
   "Returns the given parameter inside a vector if it's not a vector already."
   [p]