From 7d241fc1e212c3815ef849a3732f107224fbddd2 Mon Sep 17 00:00:00 2001 From: zkim Date: Mon, 26 Sep 2011 02:33:04 +0000 Subject: [PATCH] Updated to import clojure 1.3.0 into clojuredocs database. --- src/cd_analyzer/core.clj | 82 ++++---- src/cd_analyzer/database.clj | 368 +++++++++++++++++++---------------- 2 files changed, 246 insertions(+), 204 deletions(-) diff --git a/src/cd_analyzer/core.clj b/src/cd_analyzer/core.clj index 8f06fdd..99d5169 100644 --- a/src/cd_analyzer/core.clj +++ b/src/cd_analyzer/core.clj @@ -2,7 +2,7 @@ (:use [cd-analyzer.util] [cd-analyzer.language] [cd-analyzer.database] - [clojure.contrib.pprint]) + [clojure.pprint :only (pprint)]) (:import [java.io File FileReader] [clojure.lang LineNumberingPushbackReader])) @@ -79,7 +79,7 @@ :site-url "http://clojure.org" :copyright "© Rich Hickey. All rights reserved." :license "Eclipse Public License 1.0" - :version "1.2.0" + :version "1.3.0" :source-root (mkfile root "src" "clj")} project clojure-map project (assoc project :cljs (cljs-in (:source-root project))) @@ -157,44 +157,52 @@ (reportln) (store-lib library) (doseq [p (sort-by :name projects)] - (reportln (indt) (:name p) (pad (:name p)) "(project)")) - (reportln) - (doseq [ns (sort-by :name nss)] - (report (indt) (:name ns) (pad (:name ns)) "(ns)") - (if (store-ns-map ns) - (reportln " Ok") - (reportln " Error"))) - (reportln) - (doseq [v (sort-by :name vars)] - (report (indt) (:name v) (pad (:name v)) "(var)") - (if ((store-var-map (:name library) (:version library)) v) - (reportln " Ok") - (reportln " Error"))) - (reportln) - (doseq [v (sort-by :name vars)] - (let [v-to-vs-str (str (:name v))] - (report (indt) v-to-vs-str (pad v-to-vs-str) "(" (count (:vars-in v)) " references)")) - (if (store-var-references v) - (reportln " Ok") - (reportln " Error"))) - (reportln) - (reportln (indt) "Looking for vars to remove...") - (if (= 0 num-vars) - (reportln (indt) "No vars found, skipping removal of stale vars.") - (let [removed (remove-stale-vars (:name library) start)] - (if (= 0 (count removed)) - (reportln (indt 2) "No vars removed.") - (do - (reportln (indt 2) "Removed " (count removed) " vars:") - (doseq [vr removed] - (reportln (indt 4) (:name vr) (pad (:name vr)) "(" (:ns vr) ")")))))) - (reportln "=========================================") - (reportln (indt) num-projects " projects, " num-nss " namespaces, " num-vars " vars found in " (:name library))) + (reportln (indt) (:name p) (pad (:name p)) "(project)") + (doseq [ns (sort-by :name (:namespaces p))] + (reportln (indt) (:name ns) (pad (:name ns)) "(ns)") + (store-ns-map library ns) + (doseq [v (sort-by :name (:vars ns))] + (reportln (indt 4) (:ns v) "/" (:name v) (pad (:name v)) "(var)") + (store-var-map library ns v)))) + (reportln)) (catch Exception e (reportln "=========================================") (reportln "Import process failed: " e))) (reportln (indt 2) "Took " (/ (- (System/currentTimeMillis) start) 1000.0) "s"))) +(comment + (doseq [ns (sort-by :name nss)] + (report (indt) (:name ns) (pad (:name ns)) "(ns)") + (if (store-ns-map ns) + (reportln " Ok") + (reportln " Error"))) + (reportln) + (doseq [v (sort-by :name vars)] + (report (indt) (:name v) (pad (:name v)) "(var)") + (if ((store-var-map (:name library) (:version library)) v) + (reportln " Ok") + (reportln " Error"))) + (reportln) + (doseq [v (sort-by :name vars)] + (let [v-to-vs-str (str (:name v))] + (report (indt) v-to-vs-str (pad v-to-vs-str) "(" (count (:vars-in v)) " references)")) + (if (store-var-references v) + (reportln " Ok") + (reportln " Error"))) + (reportln) + (reportln (indt) "Looking for vars to remove...") + (if (= 0 num-vars) + (reportln (indt) "No vars found, skipping removal of stale vars.") + (let [removed (remove-stale-vars (:name library) start)] + (if (= 0 (count removed)) + (reportln (indt 2) "No vars removed.") + (do + (reportln (indt 2) "Removed " (count removed) " vars:") + (doseq [vr removed] + (reportln (indt 4) (:name vr) (pad (:name vr)) "(" (:ns vr) ")")))))) + (reportln "=========================================") + (reportln (indt) num-projects " projects, " num-nss " namespaces, " num-vars " vars found in " (:name library))) + (defn run-update [root-dir] (report-on-lib (parse-library (File. root-dir)))) @@ -204,6 +212,6 @@ (defn run-update-clojure-contrib [root-dir] (report-on-lib (parse-clojure-contrib (File. root-dir)))) -#_(run-update-clojure-core "/Users/zkim/clojurelibs/clojure") +#_(run-update-clojure-core "/home/zkim/clojure") #_(run-update-clojure-contrib "/Users/zkim/clojurelibs/clojure-contrib") -#_(pprint (parse-clojure-core (File. "/Users/zkim/clojurelibs/clojure"))) \ No newline at end of file +#_(pprint (parse-clojure-core (File. "/Users/zkim/clojurelibs/clojure"))) diff --git a/src/cd_analyzer/database.clj b/src/cd_analyzer/database.clj index f7f4a6f..9040a97 100644 --- a/src/cd_analyzer/database.clj +++ b/src/cd_analyzer/database.clj @@ -1,9 +1,10 @@ (ns cd-analyzer.database (:use [cd-analyzer.util] [cd-analyzer.language] - [clojure.contrib.pprint] - [clojure.contrib.sql] - [clojure.contrib.string :only (as-str)]) + [clojure.pprint :only (pprint)] + [clojure.contrib.string :only (as-str)]) + (:require [clojure.java.jdbc :as jdbc] + [clojure.string :as str]) (:import [java.io File])) @@ -57,8 +58,8 @@ string, which looks nasty when you display it." database spec within a transaction. Each function is passed a hash identifying the keys of the previous inserts." [db insert-fns] - (with-connection db - (transaction + (jdbc/with-connection db + (jdbc/transaction (loop [id {} todo insert-fns] (if (empty? todo) @@ -80,24 +81,6 @@ string, which looks nasty when you display it." ;;==== Functions/macros for external use ======================================= -(defn insert-record - "Equivalent of clojure.contrib.sql/insert-records that only inserts a single - record but returns the autogenerated id of that record if available." - [table record] - (let [[sql values] (sql-for-insert table record)] - (with-open [statement (.prepareStatement (connection) sql java.sql.Statement/RETURN_GENERATED_KEYS)] - (doseq [[index value] (map vector (iterate inc 1) values)] - (.setObject statement index value)) - (.execute statement) - (if-let [rs (.getGeneratedKeys statement)] - (if (.next rs) - (if-let [id (.getObject rs 1)] - id - nil) - nil) - nil)))) - - (defmacro insert-with-id "Insert records within a single transaction into the database described by the given db spec. The record format is :table { record-hash }. @@ -113,173 +96,224 @@ string, which looks nasty when you display it." `(let [insert-fns# (build-insert-fns ~table-records)] (run-chained ~db insert-fns#))) -(def *db* {:classname "com.mysql.jdbc.Driver" - :subprotocol "mysql" - :subname "//localhost:3306/clojuredocs?user=root&password=" - :create true - :username "root" - :password ""}) +(def ^{:dynamic true} *db* {:classname "com.mysql.jdbc.Driver" + :subprotocol "mysql" + :subname "//localhost:3306/clojuredocs_development?user=root&password=" + :create true + :username "root" + :password ""}) (defn query-lib-stats [libdef] - (with-connection *db* - (transaction - (let [lib-stats (with-query-results rs ["select * from libraries where name = ?" (:name libdef)] - (first rs)) - var-count (with-query-results rs ["select count(id) from functions where library = ?" (:name libdef)] - ((keyword "count(id)") (first rs)))] + (jdbc/with-connection *db* + (jdbc/transaction + (let [lib-stats (jdbc/with-query-results rs ["select * from libraries where name = ?" (:name libdef)] + (first rs))] (when lib-stats (assoc lib-stats - :var-count var-count)))))) + :var-count -1)))))) (defn insert-or-update [test-fn update-fn insert-fn] - (with-connection *db* - (transaction + (jdbc/with-connection *db* + (jdbc/transaction (let [existing (test-fn)] (if existing (update-fn existing) (insert-fn)))))) (defn sql-now [] (java.sql.Timestamp. (System/currentTimeMillis))) +(defn url-friendly [s] + (-> s + str/lower-case + (str/replace #"\s" "_"))) + (defn store-lib [libdef] (let [name (:name libdef) + version (:version libdef) description (:description libdef) site-url (:site-url libdef) source-base-url (:web-src-dir libdef) copyright (:copyright libdef) license (:license libdef) test (fn [] - (with-query-results rs ["select * from libraries where name = ? limit 1" name] + (jdbc/with-query-results rs ["select * from libraries where name = ? and version=? limit 1" name version] (first (doall rs)))) update (fn [existing] - (update-values :libraries - ["id = ?" (:id existing)] - {:description description - :site_url site-url - :source_base_url source-base-url - :copyright copyright - :license license - :updated_at (sql-now)})) + (jdbc/update-values :libraries + ["id = ?" (:id existing)] + {:description description + :site_url site-url + :source_base_url source-base-url + :copyright copyright + :license license + :version version + :updated_at (sql-now) + :url_friendly_name (url-friendly name)})) insert (fn [] - (insert-values :libraries - [:name :description :site_url :source_base_url :copyright :license :created_at :updated_at] - [name description site-url source-base-url copyright license (sql-now) (sql-now)]))] + (jdbc/insert-values :libraries + [:name + :description + :site_url + :source_base_url + :copyright + :license + :version + :url_friendly_name + :created_at + :updated_at] + [name + description + site-url + source-base-url + copyright + license + version + (url-friendly name) + (sql-now) + (sql-now)]))] (insert-or-update test update insert))) - -(defn query-var [lib var-map] - (with-connection *db* - (transaction - (with-query-results +(defn query-var [namespace-id name version] + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/with-query-results rs - ["select * from functions where library=? and ns=? and name=?" lib (str (:ns var-map)) (str (:name var-map))] + ["select * from functions where namespace_id=? and name=? and version=?" namespace-id name version] (first rs))))) -(defn store-var-map [lib version] - (fn [var-map] - (try - (let [{:keys [ns name file line arglists added doc source]} var-map] - (with-connection *db* - (transaction - (let [existing (query-var lib var-map)] - (if existing - (update-values - :functions - ["id=?" (:id existing)] - {:library lib - :version version - :ns (str ns) - :name (str name) - :file file - :line line - :arglists_comp (apply str (interpose "|" arglists)) - :added added - :doc doc - :shortdoc (if (:shortdoc existing) (:shortdoc existing) (apply str (take 70 doc))) - :source source - :updated_at (java.sql.Timestamp. (System/currentTimeMillis))}) - (insert-values - :functions - [:library :version :ns :name :file :line :arglists_comp :added :doc :shortdoc :source :updated_at :created_at] - [lib - version - (str ns) - (str name) - file - line - (apply str (interpose "|" arglists)) - added - doc - (apply str (take 70 doc)) - source - (java.sql.Timestamp. (System/currentTimeMillis)) - (java.sql.Timestamp. (System/currentTimeMillis))])))))) - (catch Exception e (println (str (:name var-map) " -- " e)))))) +(defn query-ns [ns version] + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/with-query-results + rs + ["select * from namespaces where name=? and version=?" ns version] + (first rs))))) + +(defn query-library [name version] + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/with-query-results + rs + ["select * from libraries where name=? and version=?" name version] + (first rs))))) + +(defn store-var-map [library ns var-map] + (try + (let [{:keys [ns name file line arglists added doc source]} var-map + version (:version library)] + (jdbc/with-connection *db* + (jdbc/transaction + (let [namespace (query-ns ns version) + existing (query-var (:id namespace) name version)] + (if namespace + (if existing + (jdbc/update-values + :functions + ["id=?" (:id existing)] + {:namespace_id (:id namespace) + :version version + :name (str name) + :file file + :line line + :arglists_comp (apply str (interpose "|" arglists)) + :added added + :doc doc + :shortdoc (if (:shortdoc existing) (:shortdoc existing) (apply str (take 70 doc))) + :source source + :updated_at (java.sql.Timestamp. (System/currentTimeMillis)) + :url_friendly_name (url-friendly name)}) + (jdbc/insert-values + :functions + [:namespace_id :version :name :file :line :arglists_comp :added :doc :shortdoc :source :updated_at :created_at :url_friendly_name] + [(:id namespace) + version + (str name) + file + line + (apply str (interpose "|" arglists)) + added + doc + (apply str (take 70 doc)) + source + (java.sql.Timestamp. (System/currentTimeMillis)) + (java.sql.Timestamp. (System/currentTimeMillis)) + (url-friendly name)])) + (println "Namespace" ns "not found, skipping insert of " name)))))) + (catch Exception e (println (str (:name var-map) " -- " e))))) (defn lookup-var-id [var-map] - (transaction - (with-query-results rs ["select * from functions where ns = ? and name = ? limit 1" (str (:ns var-map)) (str (:name var-map))] + (jdbc/transaction + (jdbc/with-query-results rs ["select * from functions where ns = ? and name = ? limit 1" (str (:ns var-map)) (str (:name var-map))] (:id (first rs))))) (defn remove-stale-vars [libname timestamp] - (let [to-remove - (with-connection *db* - (transaction - (with-query-results rs ["select * from functions where library = ? and (updated_at < ? or updated_at is NULL)" libname (java.sql.Timestamp. timestamp)] - (doall rs)))) - ids (map :id to-remove)] - (with-connection *db* - (transaction - (doseq [id ids] - (delete-rows :functions ["id=?" id])))) - (with-connection *db* - (transaction - (doall (map #(delete-rows :function_references ["from_function_id=?" %]) ids)) - (doall (map #(delete-rows :function_references ["to_function_id=?" %]) ids)))) - (doall (seq to-remove)))) - -(defn query-ns [ns] - (with-connection *db* - (transaction - (with-query-results rs ["select * from namespaces where name=?" ns] - (first rs))))) - -(defn store-ns-map [ns-map] - (with-connection *db* - (transaction - (let [name (str (:name ns-map)) - web-path (:web-path ns-map) - doc (:doc ns-map) - doc (remove-leading-whitespace (if doc doc "")) - existing (with-query-results rs ["select * from namespaces where name = ? limit 1" name] (first (doall rs)))] - (if existing - (update-values :namespaces - ["id=?" (:id existing)] - {:name name - :doc doc - :source_url web-path - :updated_at (sql-now)}) - (insert-values :namespaces - [:name :doc :source_url :created_at :updated_at] - [name doc web-path (sql-now) (sql-now)])))))) + #_(let [to-remove + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/with-query-results rs ["select * from functions where library = ? and (updated_at < ? or updated_at is NULL)" libname (java.sql.Timestamp. timestamp)] + (doall rs)))) + ids (map :id to-remove)] + (jdbc/with-connection *db* + (jdbc/transaction + (doseq [id ids] + (jdbc/delete-rows :functions ["id=?" id])))) + (jdbc/with-connection *db* + (jdbc/transaction + (doall (map #(jdbc/delete-rows :function_references ["from_function_id=?" %]) ids)) + (doall (map #(jdbc/delete-rows :function_references ["to_function_id=?" %]) ids)))) + (doall (seq to-remove)))) + +(defn store-ns-map [library ns-map] + (jdbc/with-connection *db* + (jdbc/transaction + (if-let [existing-lib (query-library (:name library) (:version library))] + (let [lib-id (:id existing-lib) + name (str (:name ns-map)) + web-path (:web-path ns-map) + doc (:doc ns-map) + doc (remove-leading-whitespace (if doc doc "")) + existing (jdbc/with-query-results rs ["select * from namespaces where name = ? and version=? limit 1" name (:version library)] (first (doall rs)))] + (if existing + (jdbc/update-values :namespaces + ["id=?" (:id existing)] + {:name name + :doc doc + :source_url web-path + :updated_at (sql-now) + :version (:version library) + :library_id (:id existing-lib)}) + (jdbc/insert-values :namespaces + [:name + :doc + :source_url + :created_at + :updated_at + :version + :library_id] + [name doc web-path + (sql-now) + (sql-now) + (:version library) + (:id existing-lib)]))) + (println "!! Couldn't find library in database" (:name library) (:version library)))))) (defn store-var-references [var-map] (when-let [vars-in (:vars-in var-map)] (try - (with-connection *db* - (transaction - (when-let [from-id (lookup-var-id var-map)] - (let [to-ids (map lookup-var-id vars-in)] - (doseq [to-id to-ids] - (when (not (nil? to-id)) - (let [existing (with-query-results rs - ["select * from function_references where from_function_id = ? and to_function_id = ? limit 1" from-id to-id] - (first (doall rs)))] - (when (not existing) - (insert-records :function_references {:from_function_id from-id :to_function_id to-id}))))) - true)))) - (catch Exception e - (reportln "Exception in store-var-references: ") - (reportln var-map " -> " (.getMessage e)) - nil))) + (jdbc/with-connection *db* + (jdbc/transaction + (when-let [from-id (lookup-var-id var-map)] + (let [to-ids (map lookup-var-id vars-in)] + (doseq [to-id to-ids] + (when (not (nil? to-id)) + (let [existing (jdbc/with-query-results rs + ["select * from function_references where from_function_id = ? and to_function_id = ? limit 1" from-id to-id] + (first (doall rs)))] + (when (not existing) + (jdbc/insert-records :function_references {:from_function_id from-id :to_function_id to-id}))))) + true)))) + (catch Exception e + (reportln "Exception in store-var-references: ") + (reportln var-map " -> " (.getMessage e)) + nil))) (def ccld {:name "Clojure Core" :root-dir "/Users/zkim/clojurelibs/clojure" @@ -291,11 +325,11 @@ string, which looks nasty when you display it." :license "Eclipse Public License 1.0"})) (defn track-import-start [libdef] - (with-connection *db* - (transaction - (when-let [libid (with-query-results rs ["select id from libraries where name=?", (:name libdef)] + (jdbc/with-connection *db* + (jdbc/transaction + (when-let [libid (jdbc/with-query-results rs ["select id from libraries where name=?", (:name libdef)] (:id (first rs)))] - (insert-record :library_import_tasks + (jdbc/insert-record :library_import_tasks {:library_id libid :status "RUNNING" :created_at (java.sql.Timestamp. (System/currentTimeMillis)) @@ -303,17 +337,17 @@ string, which looks nasty when you display it." (defn track-import-end [task-id lib-stats] (when (contains? lib-stats :status) - (with-connection *db* - (transaction - (update-values :library_import_tasks - ["id = ?" task-id] - (assoc lib-stats - :updated_at (java.sql.Timestamp. (System/currentTimeMillis)))))))) + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/update-values :library_import_tasks + ["id = ?" task-id] + (assoc lib-stats + :updated_at (java.sql.Timestamp. (System/currentTimeMillis)))))))) (defn import-log [task-id level message] - (with-connection *db* - (transaction - (insert-record :library_import_logs + (jdbc/with-connection *db* + (jdbc/transaction + (jdbc/insert-record :library_import_logs {:library_import_task_id task-id :level (as-str level) :message message