Stats as library

deps.edn

@@ -2,9 +2,10 @@
  :deps {com.layerware/hugsql {:mvn/version "0.4.9"}
         org.xerial/sqlite-jdbc {:mvn/version "3.20.0"}
         org.clojure/java.jdbc {:mvn/version "0.7.0"}
+        tea-time {:mvn/version "1.0.0"}
+
         ring/ring-core {:mvn/version "1.7.0-RC1"}
         ring/ring-jetty-adapter {:mvn/version "1.7.0-RC1"}
-        tea-time {:mvn/version "1.0.0"}
         metosin/reitit-core {:mvn/version "0.1.3"}
         metosin/reitit-ring {:mvn/version "0.1.3"}
         metosin/reitit-spec {:mvn/version "0.1.3"}

src/cljdoc/clojars_stats.clj

@@ -6,28 +6,35 @@
             [clojure.string :as string]
             [clojure.java.io :as io]
             [tea-time.core :as tt])
-  (:import (java.time Instant)))
+  (:import (java.time Instant Duration LocalDate)
+           (java.time.temporal ChronoUnit)))
 
 (defn stats-files []
   (->> (slurp "https://clojars.org/stats/")
        (re-seq #"downloads-\d{8}\.edn")
        (map #(str "https://clojars.org/stats/" %))
        (set)))
 
-(defn read-file [uri]
-  (let [data-dir (io/file "data")]
-    (if (.exists data-dir)
-      (let [local-file (io/file data-dir (last (string/split uri #"/")))]
-        (when-not (.exists local-file)
-          (println "Downloading" (.getName local-file))
-          (spit local-file (slurp uri)))
-        (edn/read-string (slurp local-file)))
-      (edn/read-string (slurp uri)))))
-
 (defn uri->date [uri]
   (some->> (re-find #"(\d{4})(\d{2})(\d{2})" uri)
            (rest)
-           (string/join "-")))
+           (string/join "-")
+           (LocalDate/parse)))
+
+(defn stats-for-date-exist? [db-spec date]
+  (println "Checking" date)
+  (-> (sql/query db-spec ["select exists(SELECT date FROM stats where date = ? limit 1)" date])
+      first first val pos?))
+
+(defn to-import [db-spec]
+  (let [retention-years 1
+        statsf (stats-files)]
+    (->> (zipmap (map uri->date statsf) statsf)
+         (remove #(.isBefore (key %)
+                             (.minusYears (LocalDate/now) retention-years)))
+         (remove #(stats-for-date-exist? db-spec (key %)))
+         vals
+         sort)))
 
 (defn parse-file [uri]
   (let [date (uri->date uri)]
@@ -39,7 +46,7 @@
                  :artifact_id (do (assert artifact) artifact)
                  :version (do (assert (key d)) (key d))
                  :downloads (do (assert (val d)) (val d))}))
-            (read-file uri))))
+            (edn/read-string (slurp uri)))))
 
 (defn import-file! [db f]
   (printf "Reading %s ..." f)
@@ -65,27 +72,32 @@
     (import! db (take 20 todo))
     (< 0 (count todo))))
 
+(defn start! [{:keys [db-spec]}]
+  (db/create-stats-table db-spec)
+  (db/create-imported-files-table db-spec)
+  (println "Starting background process...")
+  (tt/start!)
+  (tt/every! (* 1 60) (bound-fn [] (update! db-spec))))
+
 (defn -main [db-file]
-  (let [db {:classname   "org.sqlite.JDBC"
-            :subprotocol "sqlite"
-            :subname     db-file}]
-    (when-not (.exists (java.io.File. db-file))
-      (println "Creating tables...")
-      (db/create-stats-table db)
-      (db/create-imported-files-table db))
-    (println "Starting background process...")
-    (tt/start!)
-    (api/start! {:db db :port 3000})
-    (tt/every! (* 1 60) (bound-fn [] (update! db)))
-    (let [signal (java.util.concurrent.CountDownLatch. 1)]
-      (.await signal))))
+  (start! {:db-spec {:classname   "org.sqlite.JDBC"
+                     :subprotocol "sqlite"
+                     :subname     db-file}
+           :retention (Duration/of 365 ChronoUnit/DAYS)})
+  (let [signal (java.util.concurrent.CountDownLatch. 1)]
+    (.await signal)))
 
 (comment
   (def db {:classname   "org.sqlite.JDBC"
            :subprotocol "sqlite"
            :subname     "clojars-stats.db"})
 
   (time (-main))
+  (.minusYears (LocalDate/now) 1)
+
+  (to-import db)
+
+  (stats-for-date-exist? db (LocalDate/parse "2017-05-01"))
 
   (db/create-imported-files-table db)
   (db/create-stats-table db)