Simplify scrape and graph

data/README.md

@@ -1,6 +1,4 @@
 This directory contains the scraped data from the house and senate:
 
-> lein run :scrape-house
-
-> lein run :scrape-senate
+> lein run :scrape
 

graphs/README.md

@@ -1,4 +1,4 @@
 This directory contains the generated graphs after running:
 
-> lein run :graphs
+> lein run :graph
 

project.clj

@@ -1,8 +1,7 @@
 (defproject partisanship "1.0.0-SNAPSHOT"
   :description "Partisanship metrics"
-  :run-aliases {:scrape-house partisanship.data.scrape-house
-                :scrape-senate partisanship.data.scrape-senate
-                :graphs partisanship.graph}
+  :run-aliases {:scrape partisanship.data.scrape
+                :graph partisanship.graph}
   :dependencies [[org.clojure/clojure "1.3.0"]
                  [org.clojure/data.csv "0.1.0"]
                  [org.clojure/data.json "0.1.2"]

src/partisanship/data/scrape.clj

@@ -0,0 +1,10 @@
+(ns partisanship.data.scrape
+  (:require (partisanship.data [vote :as vote]
+                               [scrape-senate :as senate]
+                               [scrape-house :as house])))
+
+(defn -main [& args]
+  (println "Updating roll-call votes...")
+  (doall (pmap #(apply vote/update %)
+               [[senate/scraper :senate] [house/scraper :house]]))
+  (println "Finished!"))

src/partisanship/data/scrape_house.clj

@@ -11,7 +11,7 @@
 (defn pattern [res pattern]
   (map html/text (html/select res pattern)))
 
-(defn- scraper [year vote]
+(defn scraper [year vote]
   (let [res (html/html-resource (java.net.URL. (to-url year vote)))
         parties (pattern res [:party])
         yeas (butlast (pattern res [:yea-total]))
@@ -23,15 +23,10 @@
                                       :nay (Integer/parseInt nay)}})
                                   parties yeas nays))]
     (when date
-      (println "  Scraping" year vote)
+      (println "House -" year vote)
       (if (not (empty? outcome))
         {:year year
          :vote vote
          :date (tf/unparse vote/date-fmt (tf/parse in-fmt date))
          :outcome outcome}
         {}))))
-
-(defn -main [& args]
-  (println "Update house roll-call votes.  This might take a while...")
-  (vote/update scraper :house)
-  (println "Finished!"))

src/partisanship/data/scrape_senate.clj

@@ -18,7 +18,7 @@
 (defn- pattern [res pattern]
   (map html/text (html/select res pattern)))
 
-(defn- scraper [year vote]
+(defn scraper [year vote]
   (try
     (let [res (html/html-resource (java.net.URL. (to-url year vote)))
           votes (map (fn [p v] {(keyword p) {(keyword (string/lower-case v)) 1}})
@@ -29,16 +29,11 @@
                          (map (fn [[p o]] {p (select-keys o [:yea :nay])}) outcome))
           date (first (pattern res [:vote_date]))]
       (when date
-        (println "  Scraping" year vote)
+        (println "Senate -" year vote)
         (if (not (or (empty? outcome) (empty? (:D outcome)) (empty? (:R outcome))))
           {:year year
            :vote vote
            :date (tf/unparse vote/date-fmt (tf/parse in-fmt date))
            :outcome outcome}
           {})))
     (catch Exception _ nil)))
-
-(defn -main [& args]
-  (println "Update senate roll-call votes.  This might take a while...")
-  (vote/update scraper :senate)
-  (println "Finished!"))