Permalink
Browse files

Implement migration from Oracle DB

  • Loading branch information...
1 parent 04bf58a commit 7954b84f6699bb110a622d405e7cb483ce2d852f Ray Miller committed Sep 17, 2011
Showing with 177 additions and 30 deletions.
  1. +8 −2 project.clj
  2. +39 −1 src/neo_lims/core.clj
  3. +92 −2 src/neo_lims/migrate.clj
  4. +38 −25 src/neo_lims/model.clj
View
@@ -1,7 +1,13 @@
(defproject neo-lims "1.0.0-SNAPSHOT"
- :description "FIXME: write description"
+ :description "Prototype LIMS with neo4j graph database backend"
:dependencies [[org.clojure/clojure "1.2.1"]
+ [org.clojure/clojure-contrib "1.2.0"]
[borneo "0.2.0"]
[inflections "0.5.3-SNAPSHOT"]
[clojureql "1.1.0-SNAPSHOT"]
- [ojdbc/ojdbc "6"]])
+ [ojdbc/ojdbc "6"]
+ [log4j/log4j "1.2.16"]
+ [org.clojure/tools.logging "0.1.2"]
+ [clj-logging-config "1.7.0"]
+ [org.clojure/tools.cli "0.1.0"]]
+ :main neo-lims.core)
View
@@ -1 +1,39 @@
-(ns neo-lims.core)
+(ns neo-lims.core
+ (:use [clojure.tools.cli]
+ [clj-logging-config.log4j]
+ [neo-lims.migrate :only (do-migrate)])
+ (:require [borneo.core :as neo]))
+
+(defn- handle-missing-or-invalid-command
+ []
+ (println "Missing or invalid command")
+ (System/exit 1))
+
+(defn- init-logging
+ [filename debug verbose]
+ ;(set-config-logging-level! :debug)
+ (set-logger! "neo-lims"
+ :out filename
+ :level (cond debug :debug verbose :info :else :warn)
+ :pattern "%d %c %m%n"))
+
+(defn migrate
+ [args]
+ (let [{:keys [db-path log-file limit debug verbose]}
+ (cli args
+ (required ["--db-path" "Path to create neo4j database"])
+ (optional ["--log-file" "Path to write migration log"])
+ (optional ["--debug" "Log debug messages" :default false])
+ (optional ["--verbose" "Log info messages" :default true])
+ (optional ["--limit" "Maximum number of genes to process"] #(Integer. %)))]
+ (init-logging log-file debug verbose)
+ (neo/with-db! db-path (do-migrate limit))))
+
+(defn -main
+ [& args]
+ (case (first args)
+ "migrate" (migrate (rest args))
+ (handle-missing-or-invalid-command)))
+
+
+
View
@@ -1,3 +1,93 @@
(ns neo-lims.migrate
- (require [neo-lims.model :as m]
- [clojureql :as cql]))
+ (:require [clojure.tools.logging :as log]
+ [borneo.core :as neo]
+ [neo-lims.model :as m]
+ [clojure.contrib.sql :as sql]
+ [clojureql.core :as cql]))
+
+(def htgt
+ {:classname "oracle.jdbc.driver.OracleDriver"
+ :subprotocol "oracle:oci"
+ :subname "@ESMP"
+ :user "euvect_ro"
+ :password "euvect_ro"
+ :fetch-size 1000})
+
+(defn query-genes
+ []
+ (-> (cql/table :mgi_gene)
+ (cql/join (cql/table :project) (cql/where (= :project.mgi_gene_id :mgi_gene.mgi_gene_id)))
+ (cql/project [:mgi_gene.mgi_gene_id :mgi_gene.mgi_accession_id :mgi_gene.marker_symbol])
+ (cql/distinct)))
+
+(defn query-designs-for-gene
+ [mgi_gene_id]
+ (-> (cql/join (cql/table :design)
+ (cql/table :project)
+ (cql/where (= :design.design_id :project.design_id)))
+ (cql/select (cql/where (= :project.mgi_gene_id mgi_gene_id)))
+ (cql/project [:design.design_id :design.design_type])
+ (cql/distinct)))
+
+(defn query-design-wells-for-design
+ [design_id]
+ (-> (cql/table :project)
+ (cql/select (cql/where (= :project.design_id design_id)))
+ (cql/join (cql/table :well)
+ (cql/where (= :well.design_instance_id :project.design_instance_id)))
+ (cql/join (cql/table :plate)
+ (cql/where (and (= :plate.plate_id :well.plate_id)
+ (= :plate.type "DESIGN"))))
+ (cql/project [[:plate.name :as :plate_name] :well.well_name :well.well_id])
+ (cql/distinct)))
+
+(defn query-well-data-for-well
+ [well_id]
+ (-> (cql/table :well_data)
+ (cql/select (cql/where (= :well_id well_id)))
+ (cql/project [:data_type :data_value])))
+
+(defn query-child-wells-for-well
+ [parent_well_id]
+ (-> (cql/table :well)
+ (cql/select (cql/where (= :well.parent_well_id parent_well_id)))
+ (cql/join (cql/table :plate) (cql/where (= :plate.plate_id :well.plate_id)))
+ (cql/project [[:plate.name :as :plate_name] :well.well_name :well.well_id])))
+
+(defn migrate-well
+ [{:keys [plate_name well_name] :as well}]
+ (log/info (str "Migrating well: " plate_name "[" well_name "]"))
+ (let [well-node (m/create-well well)]
+ (cql/with-results [child-wells (query-child-wells-for-well (:well_id well))]
+ (doseq [child-well child-wells]
+ (let [child-well-node (migrate-well child-well)]
+ (m/associate-child-well-with-parent child-well-node well-node))))
+ well-node))
+
+(defn migrate-design
+ [design]
+ (log/info (str "Migrating design: " (:design_id design)))
+ (let [design-node (m/create-design design)]
+ (cql/with-results [design-wells (query-design-wells-for-design (:design_id design))]
+ (doseq [well design-wells]
+ (let [design-well-node (migrate-well well)]
+ (m/associate-well-with-design design-well-node design-node)))
+ design-node)))
+
+(defn migrate-gene
+ [{:keys [mgi_gene_id] :as gene}]
+ (log/info (str "Migrating gene: " mgi_gene_id))
+ (let [gene-node (m/create-gene gene)]
+ (cql/with-results [designs (query-designs-for-gene mgi_gene_id)]
+ (doseq [design designs]
+ (let [design-node (migrate-design design)]
+ (m/associate-design-with-gene design-node gene-node))))
+ gene-node))
+
+(defn do-migrate
+ [& [lim]]
+ (log/info (str "Migrating " (if lim lim "all") " genes"))
+ (sql/with-connection htgt
+ (cql/with-results [genes (query-genes)]
+ (doseq [gene (if lim (take lim genes) genes)]
+ (neo/with-tx (migrate-gene gene))))))
View
@@ -3,8 +3,6 @@
(:require [borneo.core :as neo]
[clojure.string :as str]))
-(def *db-path* "/tmp/neo-lims")
-
(defn- get-subref-node
[rel-type]
(when-let [r (neo/single-rel (neo/root) rel-type :out)]
@@ -14,11 +12,16 @@
[rel-type name]
(neo/create-child! rel-type {:name name}))
+(defn- sanitize-prop-val
+ [v]
+ (if (= (class v) (class 1M)) (long v) v))
+
(defn- create-node
[type props]
(let [store-name (plural type)
subref-rel-type (keyword (str/lower-case store-name))
- node-rel-type (keyword (str/lower-case type))]
+ node-rel-type (keyword (str/lower-case type))
+ props (zipmap (keys props) (map sanitize-prop-val (vals props)))]
(neo/with-tx
(let [subref-node (or (get-subref-node subref-rel-type)
(create-subref-node subref-rel-type store-name))]
@@ -33,8 +36,8 @@
(when (seq attrs)
(let [index (get-node-index index-name)]
(doseq [attr attrs]
- (when-let [value (get props attr)]
- (.add index node (name attr) value))))))
+ (let [value (get props attr)]
+ (when (not (nil? value)) (.add index node (name attr) value)))))))
(defn- get-single-node-via-index
[index-name key value]
@@ -54,17 +57,33 @@
(index-node-attrs "Designs" design-node design :design_id)
design-node)))
-(defn associate-design-with-gene
- [design gene]
- (neo/create-rel! design :knocks-out gene))
-
(defn- unqualified-well-name
[well-name]
(str/upper-case (subs well-name (- (count well-name) 3))))
(defn- plate-qualified-well-name
- [plate-name well-name]
- (str (str/upper-case plate-name) "_" (unqualified-well-name well-name)))
+ [{:keys [plate_name well_name]}]
+ (str (str/upper-case plate_name) "_" (unqualified-well-name well_name)))
+
+(defn create-well
+ [well]
+ (neo/with-tx
+ (let [well (assoc well :well_name (plate-qualified-well-name well))
+ well-node (create-node "Well" well)]
+ (index-node-attrs "Wells" well-node well :plate_name :well_name)
+ well-node)))
+
+(defn associate-design-with-gene
+ [design gene]
+ (neo/create-rel! design :knocks-out gene))
+
+(defn associate-well-with-design
+ [well design]
+ (neo/create-rel! well :instance-of-design design))
+
+(defn associate-child-well-with-parent
+ [child-well parent-well]
+ (neo/create-rel! child-well :child-of parent-well))
(defn get-gene-by-marker
[marker_symbol]
@@ -76,24 +95,18 @@
(defn get-design-by-id
[design_id]
- (get-single-node-via-index "Designs" "design_id" desgin_id))
+ (get-single-node-via-index "Designs" "design_id" design_id))
(defn fetch-genes
([]
(doall (map neo/props (neo/traverse (get-subref-node :genes) :gene))))
([wanted]
(doall (map neo/props (neo/traverse (get-subref-node :genes) :gene)))))
-(defn purge-genes
- []
- (neo/with-tx
- (doseq [g (neo/traverse (get-subref-node :genes) :gene)]
- (doseq [r (neo/rels g :gene)]
- (neo/delete! r))
- (neo/delete! g))))
-
-(comment
- (neo/with-db! *db-path*
- (create-gene {:mgi_accession_id "MGI:105369" :marker_symbol "Cbx1"})
- (create-gene {:mgi_accession_id "MGI:1202710" :marker_symbol "Art4"})
- (create-gene {:mgi_accession_id "MGI:1349215" :marker_symbol "Abcd1"})))
+;; (defn purge-genes
+;; []
+;; (neo/with-tx
+;; (doseq [g (neo/traverse (get-subref-node :genes) :gene)]
+;; (doseq [r (neo/rels g :gene)]
+;; (neo/delete! r))
+;; (neo/delete! g))))

0 comments on commit 7954b84

Please sign in to comment.