diff --git a/src/cd_analyzer/core.clj b/src/cd_analyzer/core.clj
index 8f06fdd..99d5169 100644
--- a/src/cd_analyzer/core.clj
+++ b/src/cd_analyzer/core.clj
@@ -2,7 +2,7 @@
(:use [cd-analyzer.util]
[cd-analyzer.language]
[cd-analyzer.database]
- [clojure.contrib.pprint])
+ [clojure.pprint :only (pprint)])
(:import [java.io File FileReader]
[clojure.lang LineNumberingPushbackReader]))
@@ -79,7 +79,7 @@
:site-url "http://clojure.org"
:copyright "© Rich Hickey. All rights reserved."
:license "Eclipse Public License 1.0"
- :version "1.2.0"
+ :version "1.3.0"
:source-root (mkfile root "src" "clj")}
project clojure-map
project (assoc project :cljs (cljs-in (:source-root project)))
@@ -157,44 +157,52 @@
(reportln)
(store-lib library)
(doseq [p (sort-by :name projects)]
- (reportln (indt) (:name p) (pad (:name p)) "(project)"))
- (reportln)
- (doseq [ns (sort-by :name nss)]
- (report (indt) (:name ns) (pad (:name ns)) "(ns)")
- (if (store-ns-map ns)
- (reportln " Ok")
- (reportln " Error")))
- (reportln)
- (doseq [v (sort-by :name vars)]
- (report (indt) (:name v) (pad (:name v)) "(var)")
- (if ((store-var-map (:name library) (:version library)) v)
- (reportln " Ok")
- (reportln " Error")))
- (reportln)
- (doseq [v (sort-by :name vars)]
- (let [v-to-vs-str (str (:name v))]
- (report (indt) v-to-vs-str (pad v-to-vs-str) "(" (count (:vars-in v)) " references)"))
- (if (store-var-references v)
- (reportln " Ok")
- (reportln " Error")))
- (reportln)
- (reportln (indt) "Looking for vars to remove...")
- (if (= 0 num-vars)
- (reportln (indt) "No vars found, skipping removal of stale vars.")
- (let [removed (remove-stale-vars (:name library) start)]
- (if (= 0 (count removed))
- (reportln (indt 2) "No vars removed.")
- (do
- (reportln (indt 2) "Removed " (count removed) " vars:")
- (doseq [vr removed]
- (reportln (indt 4) (:name vr) (pad (:name vr)) "(" (:ns vr) ")"))))))
- (reportln "=========================================")
- (reportln (indt) num-projects " projects, " num-nss " namespaces, " num-vars " vars found in " (:name library)))
+ (reportln (indt) (:name p) (pad (:name p)) "(project)")
+ (doseq [ns (sort-by :name (:namespaces p))]
+ (reportln (indt) (:name ns) (pad (:name ns)) "(ns)")
+ (store-ns-map library ns)
+ (doseq [v (sort-by :name (:vars ns))]
+ (reportln (indt 4) (:ns v) "/" (:name v) (pad (:name v)) "(var)")
+ (store-var-map library ns v))))
+ (reportln))
(catch Exception e
(reportln "=========================================")
(reportln "Import process failed: " e)))
(reportln (indt 2) "Took " (/ (- (System/currentTimeMillis) start) 1000.0) "s")))
+(comment
+ (doseq [ns (sort-by :name nss)]
+ (report (indt) (:name ns) (pad (:name ns)) "(ns)")
+ (if (store-ns-map ns)
+ (reportln " Ok")
+ (reportln " Error")))
+ (reportln)
+ (doseq [v (sort-by :name vars)]
+ (report (indt) (:name v) (pad (:name v)) "(var)")
+ (if ((store-var-map (:name library) (:version library)) v)
+ (reportln " Ok")
+ (reportln " Error")))
+ (reportln)
+ (doseq [v (sort-by :name vars)]
+ (let [v-to-vs-str (str (:name v))]
+ (report (indt) v-to-vs-str (pad v-to-vs-str) "(" (count (:vars-in v)) " references)"))
+ (if (store-var-references v)
+ (reportln " Ok")
+ (reportln " Error")))
+ (reportln)
+ (reportln (indt) "Looking for vars to remove...")
+ (if (= 0 num-vars)
+ (reportln (indt) "No vars found, skipping removal of stale vars.")
+ (let [removed (remove-stale-vars (:name library) start)]
+ (if (= 0 (count removed))
+ (reportln (indt 2) "No vars removed.")
+ (do
+ (reportln (indt 2) "Removed " (count removed) " vars:")
+ (doseq [vr removed]
+ (reportln (indt 4) (:name vr) (pad (:name vr)) "(" (:ns vr) ")"))))))
+ (reportln "=========================================")
+ (reportln (indt) num-projects " projects, " num-nss " namespaces, " num-vars " vars found in " (:name library)))
+
(defn run-update [root-dir]
(report-on-lib (parse-library (File. root-dir))))
@@ -204,6 +212,6 @@
(defn run-update-clojure-contrib [root-dir]
(report-on-lib (parse-clojure-contrib (File. root-dir))))
-#_(run-update-clojure-core "/Users/zkim/clojurelibs/clojure")
+#_(run-update-clojure-core "/home/zkim/clojure")
#_(run-update-clojure-contrib "/Users/zkim/clojurelibs/clojure-contrib")
-#_(pprint (parse-clojure-core (File. "/Users/zkim/clojurelibs/clojure")))
\ No newline at end of file
+#_(pprint (parse-clojure-core (File. "/Users/zkim/clojurelibs/clojure")))
diff --git a/src/cd_analyzer/database.clj b/src/cd_analyzer/database.clj
index f7f4a6f..9040a97 100644
--- a/src/cd_analyzer/database.clj
+++ b/src/cd_analyzer/database.clj
@@ -1,9 +1,10 @@
(ns cd-analyzer.database
(:use [cd-analyzer.util]
[cd-analyzer.language]
- [clojure.contrib.pprint]
- [clojure.contrib.sql]
- [clojure.contrib.string :only (as-str)])
+ [clojure.pprint :only (pprint)]
+ [clojure.contrib.string :only (as-str)])
+ (:require [clojure.java.jdbc :as jdbc]
+ [clojure.string :as str])
(:import [java.io File]))
@@ -57,8 +58,8 @@ string, which looks nasty when you display it."
database spec within a transaction. Each function is passed a hash
identifying the keys of the previous inserts."
[db insert-fns]
- (with-connection db
- (transaction
+ (jdbc/with-connection db
+ (jdbc/transaction
(loop [id {}
todo insert-fns]
(if (empty? todo)
@@ -80,24 +81,6 @@ string, which looks nasty when you display it."
;;==== Functions/macros for external use =======================================
-(defn insert-record
- "Equivalent of clojure.contrib.sql/insert-records that only inserts a single
- record but returns the autogenerated id of that record if available."
- [table record]
- (let [[sql values] (sql-for-insert table record)]
- (with-open [statement (.prepareStatement (connection) sql java.sql.Statement/RETURN_GENERATED_KEYS)]
- (doseq [[index value] (map vector (iterate inc 1) values)]
- (.setObject statement index value))
- (.execute statement)
- (if-let [rs (.getGeneratedKeys statement)]
- (if (.next rs)
- (if-let [id (.getObject rs 1)]
- id
- nil)
- nil)
- nil))))
-
-
(defmacro insert-with-id
"Insert records within a single transaction into the database described by
the given db spec. The record format is :table { record-hash }.
@@ -113,173 +96,224 @@ string, which looks nasty when you display it."
`(let [insert-fns# (build-insert-fns ~table-records)]
(run-chained ~db insert-fns#)))
-(def *db* {:classname "com.mysql.jdbc.Driver"
- :subprotocol "mysql"
- :subname "//localhost:3306/clojuredocs?user=root&password="
- :create true
- :username "root"
- :password ""})
+(def ^{:dynamic true} *db* {:classname "com.mysql.jdbc.Driver"
+ :subprotocol "mysql"
+ :subname "//localhost:3306/clojuredocs_development?user=root&password="
+ :create true
+ :username "root"
+ :password ""})
(defn query-lib-stats [libdef]
- (with-connection *db*
- (transaction
- (let [lib-stats (with-query-results rs ["select * from libraries where name = ?" (:name libdef)]
- (first rs))
- var-count (with-query-results rs ["select count(id) from functions where library = ?" (:name libdef)]
- ((keyword "count(id)") (first rs)))]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (let [lib-stats (jdbc/with-query-results rs ["select * from libraries where name = ?" (:name libdef)]
+ (first rs))]
(when lib-stats (assoc lib-stats
- :var-count var-count))))))
+ :var-count -1))))))
(defn insert-or-update
[test-fn update-fn insert-fn]
- (with-connection *db*
- (transaction
+ (jdbc/with-connection *db*
+ (jdbc/transaction
(let [existing (test-fn)]
(if existing (update-fn existing) (insert-fn))))))
(defn sql-now [] (java.sql.Timestamp. (System/currentTimeMillis)))
+(defn url-friendly [s]
+ (-> s
+ str/lower-case
+ (str/replace #"\s" "_")))
+
(defn store-lib [libdef]
(let [name (:name libdef)
+ version (:version libdef)
description (:description libdef)
site-url (:site-url libdef)
source-base-url (:web-src-dir libdef)
copyright (:copyright libdef)
license (:license libdef)
test (fn []
- (with-query-results rs ["select * from libraries where name = ? limit 1" name]
+ (jdbc/with-query-results rs ["select * from libraries where name = ? and version=? limit 1" name version]
(first (doall rs))))
update (fn [existing]
- (update-values :libraries
- ["id = ?" (:id existing)]
- {:description description
- :site_url site-url
- :source_base_url source-base-url
- :copyright copyright
- :license license
- :updated_at (sql-now)}))
+ (jdbc/update-values :libraries
+ ["id = ?" (:id existing)]
+ {:description description
+ :site_url site-url
+ :source_base_url source-base-url
+ :copyright copyright
+ :license license
+ :version version
+ :updated_at (sql-now)
+ :url_friendly_name (url-friendly name)}))
insert (fn []
- (insert-values :libraries
- [:name :description :site_url :source_base_url :copyright :license :created_at :updated_at]
- [name description site-url source-base-url copyright license (sql-now) (sql-now)]))]
+ (jdbc/insert-values :libraries
+ [:name
+ :description
+ :site_url
+ :source_base_url
+ :copyright
+ :license
+ :version
+ :url_friendly_name
+ :created_at
+ :updated_at]
+ [name
+ description
+ site-url
+ source-base-url
+ copyright
+ license
+ version
+ (url-friendly name)
+ (sql-now)
+ (sql-now)]))]
(insert-or-update test update insert)))
-
-(defn query-var [lib var-map]
- (with-connection *db*
- (transaction
- (with-query-results
+(defn query-var [namespace-id name version]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/with-query-results
rs
- ["select * from functions where library=? and ns=? and name=?" lib (str (:ns var-map)) (str (:name var-map))]
+ ["select * from functions where namespace_id=? and name=? and version=?" namespace-id name version]
(first rs)))))
-(defn store-var-map [lib version]
- (fn [var-map]
- (try
- (let [{:keys [ns name file line arglists added doc source]} var-map]
- (with-connection *db*
- (transaction
- (let [existing (query-var lib var-map)]
- (if existing
- (update-values
- :functions
- ["id=?" (:id existing)]
- {:library lib
- :version version
- :ns (str ns)
- :name (str name)
- :file file
- :line line
- :arglists_comp (apply str (interpose "|" arglists))
- :added added
- :doc doc
- :shortdoc (if (:shortdoc existing) (:shortdoc existing) (apply str (take 70 doc)))
- :source source
- :updated_at (java.sql.Timestamp. (System/currentTimeMillis))})
- (insert-values
- :functions
- [:library :version :ns :name :file :line :arglists_comp :added :doc :shortdoc :source :updated_at :created_at]
- [lib
- version
- (str ns)
- (str name)
- file
- line
- (apply str (interpose "|" arglists))
- added
- doc
- (apply str (take 70 doc))
- source
- (java.sql.Timestamp. (System/currentTimeMillis))
- (java.sql.Timestamp. (System/currentTimeMillis))]))))))
- (catch Exception e (println (str (:name var-map) " -- " e))))))
+(defn query-ns [ns version]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/with-query-results
+ rs
+ ["select * from namespaces where name=? and version=?" ns version]
+ (first rs)))))
+
+(defn query-library [name version]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/with-query-results
+ rs
+ ["select * from libraries where name=? and version=?" name version]
+ (first rs)))))
+
+(defn store-var-map [library ns var-map]
+ (try
+ (let [{:keys [ns name file line arglists added doc source]} var-map
+ version (:version library)]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (let [namespace (query-ns ns version)
+ existing (query-var (:id namespace) name version)]
+ (if namespace
+ (if existing
+ (jdbc/update-values
+ :functions
+ ["id=?" (:id existing)]
+ {:namespace_id (:id namespace)
+ :version version
+ :name (str name)
+ :file file
+ :line line
+ :arglists_comp (apply str (interpose "|" arglists))
+ :added added
+ :doc doc
+ :shortdoc (if (:shortdoc existing) (:shortdoc existing) (apply str (take 70 doc)))
+ :source source
+ :updated_at (java.sql.Timestamp. (System/currentTimeMillis))
+ :url_friendly_name (url-friendly name)})
+ (jdbc/insert-values
+ :functions
+ [:namespace_id :version :name :file :line :arglists_comp :added :doc :shortdoc :source :updated_at :created_at :url_friendly_name]
+ [(:id namespace)
+ version
+ (str name)
+ file
+ line
+ (apply str (interpose "|" arglists))
+ added
+ doc
+ (apply str (take 70 doc))
+ source
+ (java.sql.Timestamp. (System/currentTimeMillis))
+ (java.sql.Timestamp. (System/currentTimeMillis))
+ (url-friendly name)]))
+ (println "Namespace" ns "not found, skipping insert of " name))))))
+ (catch Exception e (println (str (:name var-map) " -- " e)))))
(defn lookup-var-id [var-map]
- (transaction
- (with-query-results rs ["select * from functions where ns = ? and name = ? limit 1" (str (:ns var-map)) (str (:name var-map))]
+ (jdbc/transaction
+ (jdbc/with-query-results rs ["select * from functions where ns = ? and name = ? limit 1" (str (:ns var-map)) (str (:name var-map))]
(:id (first rs)))))
(defn remove-stale-vars [libname timestamp]
- (let [to-remove
- (with-connection *db*
- (transaction
- (with-query-results rs ["select * from functions where library = ? and (updated_at < ? or updated_at is NULL)" libname (java.sql.Timestamp. timestamp)]
- (doall rs))))
- ids (map :id to-remove)]
- (with-connection *db*
- (transaction
- (doseq [id ids]
- (delete-rows :functions ["id=?" id]))))
- (with-connection *db*
- (transaction
- (doall (map #(delete-rows :function_references ["from_function_id=?" %]) ids))
- (doall (map #(delete-rows :function_references ["to_function_id=?" %]) ids))))
- (doall (seq to-remove))))
-
-(defn query-ns [ns]
- (with-connection *db*
- (transaction
- (with-query-results rs ["select * from namespaces where name=?" ns]
- (first rs)))))
-
-(defn store-ns-map [ns-map]
- (with-connection *db*
- (transaction
- (let [name (str (:name ns-map))
- web-path (:web-path ns-map)
- doc (:doc ns-map)
- doc (remove-leading-whitespace (if doc doc ""))
- existing (with-query-results rs ["select * from namespaces where name = ? limit 1" name] (first (doall rs)))]
- (if existing
- (update-values :namespaces
- ["id=?" (:id existing)]
- {:name name
- :doc doc
- :source_url web-path
- :updated_at (sql-now)})
- (insert-values :namespaces
- [:name :doc :source_url :created_at :updated_at]
- [name doc web-path (sql-now) (sql-now)]))))))
+ #_(let [to-remove
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/with-query-results rs ["select * from functions where library = ? and (updated_at < ? or updated_at is NULL)" libname (java.sql.Timestamp. timestamp)]
+ (doall rs))))
+ ids (map :id to-remove)]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (doseq [id ids]
+ (jdbc/delete-rows :functions ["id=?" id]))))
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (doall (map #(jdbc/delete-rows :function_references ["from_function_id=?" %]) ids))
+ (doall (map #(jdbc/delete-rows :function_references ["to_function_id=?" %]) ids))))
+ (doall (seq to-remove))))
+
+(defn store-ns-map [library ns-map]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (if-let [existing-lib (query-library (:name library) (:version library))]
+ (let [lib-id (:id existing-lib)
+ name (str (:name ns-map))
+ web-path (:web-path ns-map)
+ doc (:doc ns-map)
+ doc (remove-leading-whitespace (if doc doc ""))
+ existing (jdbc/with-query-results rs ["select * from namespaces where name = ? and version=? limit 1" name (:version library)] (first (doall rs)))]
+ (if existing
+ (jdbc/update-values :namespaces
+ ["id=?" (:id existing)]
+ {:name name
+ :doc doc
+ :source_url web-path
+ :updated_at (sql-now)
+ :version (:version library)
+ :library_id (:id existing-lib)})
+ (jdbc/insert-values :namespaces
+ [:name
+ :doc
+ :source_url
+ :created_at
+ :updated_at
+ :version
+ :library_id]
+ [name doc web-path
+ (sql-now)
+ (sql-now)
+ (:version library)
+ (:id existing-lib)])))
+ (println "!! Couldn't find library in database" (:name library) (:version library))))))
(defn store-var-references [var-map]
(when-let [vars-in (:vars-in var-map)]
(try
- (with-connection *db*
- (transaction
- (when-let [from-id (lookup-var-id var-map)]
- (let [to-ids (map lookup-var-id vars-in)]
- (doseq [to-id to-ids]
- (when (not (nil? to-id))
- (let [existing (with-query-results rs
- ["select * from function_references where from_function_id = ? and to_function_id = ? limit 1" from-id to-id]
- (first (doall rs)))]
- (when (not existing)
- (insert-records :function_references {:from_function_id from-id :to_function_id to-id})))))
- true))))
- (catch Exception e
- (reportln "Exception in store-var-references: ")
- (reportln var-map " -> " (.getMessage e))
- nil)))
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (when-let [from-id (lookup-var-id var-map)]
+ (let [to-ids (map lookup-var-id vars-in)]
+ (doseq [to-id to-ids]
+ (when (not (nil? to-id))
+ (let [existing (jdbc/with-query-results rs
+ ["select * from function_references where from_function_id = ? and to_function_id = ? limit 1" from-id to-id]
+ (first (doall rs)))]
+ (when (not existing)
+ (jdbc/insert-records :function_references {:from_function_id from-id :to_function_id to-id})))))
+ true))))
+ (catch Exception e
+ (reportln "Exception in store-var-references: ")
+ (reportln var-map " -> " (.getMessage e))
+ nil)))
(def ccld {:name "Clojure Core"
:root-dir "/Users/zkim/clojurelibs/clojure"
@@ -291,11 +325,11 @@ string, which looks nasty when you display it."
:license "Eclipse Public License 1.0"}))
(defn track-import-start [libdef]
- (with-connection *db*
- (transaction
- (when-let [libid (with-query-results rs ["select id from libraries where name=?", (:name libdef)]
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (when-let [libid (jdbc/with-query-results rs ["select id from libraries where name=?", (:name libdef)]
(:id (first rs)))]
- (insert-record :library_import_tasks
+ (jdbc/insert-record :library_import_tasks
{:library_id libid
:status "RUNNING"
:created_at (java.sql.Timestamp. (System/currentTimeMillis))
@@ -303,17 +337,17 @@ string, which looks nasty when you display it."
(defn track-import-end [task-id lib-stats]
(when (contains? lib-stats :status)
- (with-connection *db*
- (transaction
- (update-values :library_import_tasks
- ["id = ?" task-id]
- (assoc lib-stats
- :updated_at (java.sql.Timestamp. (System/currentTimeMillis))))))))
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/update-values :library_import_tasks
+ ["id = ?" task-id]
+ (assoc lib-stats
+ :updated_at (java.sql.Timestamp. (System/currentTimeMillis))))))))
(defn import-log [task-id level message]
- (with-connection *db*
- (transaction
- (insert-record :library_import_logs
+ (jdbc/with-connection *db*
+ (jdbc/transaction
+ (jdbc/insert-record :library_import_logs
{:library_import_task_id task-id
:level (as-str level)
:message message