basic analysis

README.md

@@ -28,7 +28,13 @@ This will create a db called `git` (you can call it whatever you like) and impor
     Importing commit: 6bbfd943766e11e52a3fe21b177d55536892d132
     Import complete!
 
-The import is not too peppy, since it shells to `git` relentlessly, but it imports e.g. Clojure's entire commit history in about 10 minutes.
+    Analyzing...
+    Running analyzer: :clj on [.clj]
+    analyzing file: 17592186045504
+    analyzing file: 17592186045496
+    Analysis complete!
+
+The import is not too peppy, since it shells to `git` relentlessly, but it imports e.g. Clojure's entire commit history in about 10 minutes, plus analysis (not yet timed).
 
 You can import more than one repo into the same db. You can re-import later after some more commits and they will be incrementally added.
 

project.clj

@@ -6,5 +6,4 @@
   :main datomic.codeq.core
   :dependencies [[com.datomic/datomic-free "0.8.3538"]
                  [commons-codec "1.7"]
-                 [local.repo/clojure "1.5.0-alpha6"]
-                 #_[org.clojure/clojure "1.5.0-alpha6"]])
+                 [org.clojure/clojure "1.5.0-alpha6"]])

src/datomic/codeq/analyzers/clj.clj

@@ -13,18 +13,51 @@
 
 (defn analyze-1
   "returns [tx-data ctx]"
-  [db f x loc seg ret {:keys [sha->id added ns] :as ctx}]
+  [db f x loc seg ret {:keys [sha->id codename->id added ns] :as ctx}]
   (if loc
     (let [sha (-> seg az/ws-minify az/sha)
-          cid (sha->id sha)
-          newcid (and (tempid? cid) (not (added cid)))
-          ret (cond-> ret newcid (conj {:db/id cid :code/sha sha :code/text seg}))
-          added (cond-> added newcid (conj cid))]
+          codeid (sha->id sha)
+          newcodeid (and (tempid? codeid) (not (added codeid)))
+          ret (cond-> ret newcodeid (conj {:db/id codeid :code/sha sha :code/text seg}))
+          added (cond-> added newcodeid (conj codeid))
+
+          codeqid (or (ffirst (d/q '[:find ?e :in $ ?f ?loc
+                                     :where [?e :codeq/file ?f] [?e :codeq/loc ?loc]]
+                                   db f loc))
+                      (d/tempid :db.part/user))
+
+          op (first x)
+          ns? (= op 'ns)
+          defing (and ns
+                      (symbol? op)
+                      (#{'def 'defn} op))
+
+          naming (cond 
+                  ns? (str (second x))
+                  defing (str (symbol (name ns) (name (second x)))))
+
+          nameid (when naming (codename->id naming))
+
+          ret (cond-> ret 
+                      (tempid? codeqid)
+                      (conj {:db/id codeqid
+                             :codeq/file f
+                             :codeq/loc loc
+                             :codeq/code codeid})
+
+                      ns?
+                      (conj [:db/add codeqid :clj/ns nameid])
+
+                      defing
+                      (conj [:db/add codeqid :clj/def nameid])
+
+                      (tempid? nameid)
+                      (conj [:db/add nameid :code/name naming]))]
       [ret (assoc ctx :added added)])
     [ret ctx]))
 
 (defn analyze 
-   [a db f src]
+   [db f src]
    (with-open [r (clojure.lang.LineNumberingPushbackReader. (java.io.StringReader. src))]
      (let [loffs (az/line-offsets src)
            eof (Object.)
@@ -46,23 +79,26 @@
                  [ret ctx] (analyze-1 db f x loc seg ret ctx)]
              (recur ret ctx (read r false eof))))))))
 
+(defn schemas []
+  {1 [{:db/id #db/id[:db.part/db]
+       :db/ident :clj/ns
+       :db/valueType :db.type/ref
+       :db/cardinality :db.cardinality/one
+       :db/doc "codename of ns defined by expression"
+       :db.install/_attribute :db.part/db}
+      {:db/id #db/id[:db.part/db]
+       :db/ident :clj/def
+       :db/valueType :db.type/ref
+       :db/cardinality :db.cardinality/one
+       :db/doc "codename defined by expression"
+       :db.install/_attribute :db.part/db}]})
+
 (deftype CljAnalyzer []
   az/Analyzer
   (keyname [a] :clj)
   (revision [a] 1)
   (extensions [a] [".clj"])
-  (schemas [a] {1 [{:db/id #db/id[:db.part/db]
-                    :db/ident :clj/ns
-                    :db/valueType :db.type/ref
-                    :db/cardinality :db.cardinality/one
-                    :db/doc "codename of ns defined by expression"
-                    :db.install/_attribute :db.part/db}
-                   {:db/id #db/id[:db.part/db]
-                    :db/ident :clj/def
-                    :db/valueType :db.type/ref
-                    :db/cardinality :db.cardinality/one
-                    :db/doc "codename defined by expression"
-                    :db.install/_attribute :db.part/db}]})
-  (analyze [a db f src] (analyze a db f src)))
+  (schemas [a] (schemas))
+  (analyze [a db f src] (analyze db f src)))
 
 (defn impl [] (CljAnalyzer.))

src/datomic/codeq/core.clj

@@ -550,9 +550,20 @@
 (seq (d/datoms db :aevt :git/message))
 (seq (d/datoms db :aevt :codeq/file))
 (count (seq (d/datoms db :aevt :code/sha)))
+(take 20 (seq (d/datoms db :aevt :code/text)))
+(seq (d/datoms db :aevt :code/name))
+(count (seq (d/datoms db :aevt :codeq/code)))
 (d/q '[:find ?e :where [?f :file/name "core.clj"] [?n :git/filename ?f] [?n :git/object ?e]] db)
-(d/q '[:find ?m :where [_ :git/message ?m] [(.contains ?m "\n")]] db)
-(d/q '[:find ?m :where [_ :code/text ?m] [(.contains ?m "(ns ")]] db)
+(d/q '[:find ?m :where [_ :git/message ?m] [(.contains ^String ?m "\n")]] db)
+(d/q '[:find ?m :where [_ :code/text ?m] [(.contains ^String ?m "(ns ")]] db)
+(sort (d/q '[:find ?var ?def :where [?cn :code/name ?var] [?cq :clj/def ?cn] [?cq :codeq/code ?def]] db))
+(sort (d/q '[:find ?var ?def :where [?cn :code/name ?var] [?cq :clj/ns ?cn] [?cq :codeq/code ?def]] db))
+(sort (d/q '[:find ?var ?def ?n :where 
+             [?cn :code/name ?var] 
+             [?cq :clj/ns ?cn]
+             [?cq :codeq/file ?f]
+             [?n :git/object ?f]
+             [?cq :codeq/code ?def]] db))
 (def x "(doseq [f (clojure.set/difference cfiles afiles)]
           ;;analyze them
           (println \"analyzing file:\" f)