Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Bufix handling of arguments when --limit not specified on the

command-line.

Move some string manipulation functions into the data-helpers
namespace.

Introduce a sentinel to enable thread-safe find-or-create plate
function.

Create plates for each well as it is migrated, and add an :on-plate
relation to the well instead of indexing :plate_name.
  • Loading branch information...
commit fc64f2ae485fbc9422bd8af303b2f32415de4a78 1 parent d4abeef
Ray Miller authored
View
2  src/neo_lims/core.clj
@@ -25,7 +25,7 @@
(optional ["--log-file" "Path to write migration log"])
(optional ["--debug" "Log debug messages" :default false])
(optional ["--verbose" "Log info messages" :default true])
- (optional ["--limit" "Maximum number of genes to process"] #(Integer. %)))]
+ (optional ["--limit" "Maximum number of genes to process" :default 0] #(Integer. %)))]
(init-logging log-file debug verbose)
(neo/with-db! db-path (do-migrate limit))))
View
20 src/neo_lims/data_helpers.clj
@@ -0,0 +1,20 @@
+(ns neo-lims.data-helpers
+ (:require [clojure.string :as str]))
+
+(defn unqualified-well-name
+ [well-name]
+ (str/upper-case (subs well-name (- (count well-name) 3))))
+
+(defn plate-qualified-well-name
+ [{:keys [plate_name well_name]}]
+ (str (str/upper-case plate_name) "_" (unqualified-well-name well_name)))
+
+(defn sanitize-key-name
+ [k]
+ (-> k
+ str/trim
+ (str/replace #"\s+" "_")
+ (str/replace #"\W" "")
+ str/lower-case
+ keyword))
+
View
60 src/neo_lims/migrate.clj
@@ -1,14 +1,15 @@
(ns neo-lims.migrate
(:require [clojure.tools.logging :as log]
[borneo.core :as neo]
- [neo-lims.model :as m]
+ [neo-lims.model :as m]
+ [neo-lims.data-helpers :as dh]
[clojure.contrib.sql :as sql]
[clojureql.core :as cql]))
(def htgt
{:classname "oracle.jdbc.driver.OracleDriver"
:subprotocol "oracle:oci"
- :subname "@ESMP"
+ :subname "@ESMT"
:user "euvect_ro"
:password "euvect_ro"
:fetch-size 1000})
@@ -38,26 +39,53 @@
(cql/join (cql/table :plate)
(cql/where (and (= :plate.plate_id :well.plate_id)
(= :plate.type "DESIGN"))))
- (cql/project [[:plate.name :as :plate_name] :well.well_name :well.well_id])
+ (cql/project [[:plate.name :as :plate_name] [:plate.type :as :plate_type]
+ :well.plate_id :well.well_name :well.well_id])
(cql/distinct)))
-(defn query-well-data-for-well
- [well_id]
- (-> (cql/table :well_data)
- (cql/select (cql/where (= :well_id well_id)))
- (cql/project [:data_type :data_value])))
-
(defn query-child-wells-for-well
[parent_well_id]
(-> (cql/table :well)
(cql/select (cql/where (= :well.parent_well_id parent_well_id)))
(cql/join (cql/table :plate) (cql/where (= :plate.plate_id :well.plate_id)))
- (cql/project [[:plate.name :as :plate_name] :well.well_name :well.well_id])))
+ (cql/project [[:plate.name :as :plate_name] [:plate.type :as :plate_type]
+ :well.plate_id :well.well_name :well.well_id])))
+
+(defn get-associated-data
+ "Get plate_data (well_data) for plate (well) with plate_id (well_id) id, according to type"
+ [type id]
+ (let [table-name (keyword (str type "_data"))
+ key-name (keyword (str type "_id"))
+ query (-> (cql/table table-name)
+ (cql/select (cql/where (= key-name id)))
+ (cql/project [:data_type :data_value]))]
+ (cql/with-results [data query]
+ (zipmap (map #(dh/sanitize-key-name (:data_type %)) data)
+ (map :data_value data)))))
+
+(defn get-well-props
+ [well]
+ (-> (get-associated-data "well" (:well_id well))
+ (assoc :well_name (dh/plate-qualified-well-name well))
+ (dissoc :well_id)))
+
+(defn get-plate-props
+ [{:keys [plate_name plate_type plate_id]}]
+ (-> (get-associated-data "plate" plate_id)
+ (assoc :plate_name plate_name :plate_type plate_type)))
+
+(defn find-or-create-plate
+ [{:keys [plate_name] :as plate}]
+ (m/with-plate-lock
+ (if-let [plate-node (m/get-plate-by-name plate_name)]
+ plate-node
+ (m/create-plate (get-plate-props plate)))))
(defn migrate-well
- [{:keys [plate_name well_name] :as well}]
- (log/info (str "Migrating well: " plate_name "[" well_name "]"))
- (let [well-node (m/create-well well)]
+ [{:keys [plate_name plate_type well_name] :as well}]
+ (log/info (str "Migrating well: " plate_name "[" well_name "]"))
+ (let [plate-node (find-or-create-plate well)
+ well-node (m/create-well plate-node (get-well-props well))]
(cql/with-results [child-wells (query-child-wells-for-well (:well_id well))]
(doseq [child-well child-wells]
(let [child-well-node (migrate-well child-well)]
@@ -85,9 +113,9 @@
gene-node))
(defn do-migrate
- [& [lim]]
- (log/info (str "Migrating " (if lim lim "all") " genes"))
+ [lim]
+ (log/info (str "Migrating " (if (> lim 0) lim "all") " genes"))
(sql/with-connection htgt
(cql/with-results [genes (query-genes)]
- (doseq [gene (if lim (take lim genes) genes)]
+ (doseq [gene (if (> lim 0) (take lim genes) genes)]
(neo/with-tx (migrate-gene gene))))))
View
35 src/neo_lims/model.clj
@@ -3,6 +3,14 @@
(:require [borneo.core :as neo]
[clojure.string :as str]))
+;; Neo4j transaction level is read-committed, so find-or-create
+;; operations need an explicit lock.
+(def plate-sentinel (Object.))
+
+(defmacro with-plate-lock
+ [& body]
+ `(locking plate-sentinel ~@body))
+
(defn- get-subref-node
[rel-type]
(when-let [r (neo/single-rel (neo/root) rel-type :out)]
@@ -59,22 +67,21 @@
(index-node-attrs "Designs" design-node design :design_id)
design-node)))
-(defn- unqualified-well-name
- [well-name]
- (str/upper-case (subs well-name (- (count well-name) 3))))
-
-(defn- plate-qualified-well-name
- [{:keys [plate_name well_name]}]
- (str (str/upper-case plate_name) "_" (unqualified-well-name well_name)))
-
(defn create-well
- [well]
+ [plate-node well]
(neo/with-tx
- (let [well (assoc well :well_name (plate-qualified-well-name well))
- well-node (create-node "Well" well)]
- (index-node-attrs "Wells" well-node well :plate_name :well_name)
+ (let [well-node (create-node "Well" well)]
+ (neo/create-rel! well-node :on-plate plate-node)
+ (index-node-attrs "Wells" well-node well :well_name)
well-node)))
+(defn create-plate
+ [plate]
+ (neo/with-tx
+ (let [plate-node (create-node "Plate" plate)]
+ (index-node-attrs "Plates" plate-node plate :plate_name)
+ plate-node)))
+
(defn associate-design-with-gene
[design gene]
(neo/create-rel! design :knocks-out gene))
@@ -108,6 +115,10 @@
[well_name]
(get-single-node-via-index "Wells" "well_name" well_name))
+(defn get-plate-by-name
+ [plate_name]
+ (get-single-node-via-index "Plates" "plate_name" plate_name))
+
(defn get-ancestors-of-well
[well_name]
(when-let [well-node (get-well-by-name well_name)]
Please sign in to comment.
Something went wrong with that request. Please try again.