Permalink
Browse files

Add ability to query a local snapshot of clojuredocs API results

without using the Internet at all.  The file test/partial-snapshot.clj
can be used to test this capability on a partial snapshot of
clojuredocs contents.  See comments in that file for how to use it.
  • Loading branch information...
1 parent 579ce05 commit a2d5e4a1eac093f4c57cf8c7c2e04441f7f84898 @jafingerhut committed Mar 7, 2011
Showing with 636 additions and 8 deletions.
  1. +123 −8 src/cd_client/core.clj
  2. +513 −0 test/partial-snapshot.clj
View
@@ -2,7 +2,8 @@
(:require [org.danlarkin.json :as json]
[clj-http.client :as http]
[clojure.string :as string])
- (:use [clojure.java.browse :only [browse-url]]))
+ (:use [clojure.java.browse :only [browse-url]]
+ [clojure.pprint :only [pprint]]))
; For testing purposes use localhost:8080
@@ -15,6 +16,41 @@
(def *seealso-api* (str *clojuredocs-root* "/see-also/"))
+;; Use one of the functions set-local-mode! or set-web-mode! below to
+;; change the mode, and show-mode to show the current mode.
+(def *cd-client-mode* (ref {:source :web}))
+
+
+(defn set-local-mode! [fname]
+ ;; TBD: Handle errors in attempting to open the file, or as returned
+ ;; from read.
+ (let [data (with-open [s (java.io.PushbackReader.
+ (java.io.InputStreamReader.
+ (java.io.FileInputStream.
+ (java.io.File. fname))))]
+ (read s))]
+ (dosync (alter *cd-client-mode*
+ (fn [cur-val]
+ {:source :local-file, :filename fname, :data data})))
+ (println "Read info on" (count data) "names from local file")
+ (println fname)))
+
+
+(defn set-web-mode! []
+ (dosync (alter *cd-client-mode* (fn [cur-val] {:source :web})))
+ (println "Now retrieving clojuredocs data from web site clojuredocs.org"))
+
+
+(defn show-mode []
+ (let [mode @*cd-client-mode*]
+ (if (= :web (:source mode))
+ (println "Web mode. Data is retrieved from clojuredocs.org")
+ (do
+ (println "Local mode. Data for" (count (:data mode))
+ "names was retrieved from the file")
+ (println (:filename mode))))))
+
+
(defn- fixup-name-url
"Replace some special characters in symbol names in order to construct a URL that works on clojuredocs.org"
[name]
@@ -77,8 +113,18 @@
(defn examples-core
"Return examples from clojuredocs for a given namespace and name (as strings)"
[ns name]
- (json/decode-from-str (:body (http/get (str *examples-api* ns "/"
- (fixup-name-url name))))))
+ (let [mode @*cd-client-mode*]
+ (if (= :web (:source mode))
+ (json/decode-from-str (:body (http/get (str *examples-api* ns "/"
+ (fixup-name-url name)))))
+ ;; Make examples-core return the value that I wish the
+ ;; json/decode-from-str call above did when there are no
+ ;; examples, i.e. the URL and an empty vector of examples. Then
+ ;; I can test browse-to to see if it will work unmodified for
+ ;; names that have no examples.
+ (let [name-info (get (:data mode) (str ns "/" name))]
+ {:examples (:examples name-info),
+ :url (:url name-info)}))))
(defmacro examples
@@ -116,6 +162,8 @@
`(pr-examples-core ~ns ~name)))
+;; TBD: Think about how to implement search when in local mode.
+
(defn search
"Search for a method name within an (optional) namespace"
([name]
@@ -127,8 +175,12 @@
(defn comments-core
"Return comments from clojuredocs for a given namespace and name (as strings)"
[ns name]
- (json/decode-from-str (:body (http/get (str *comments-api* ns "/"
- (fixup-name-url name))))))
+ (let [mode @*cd-client-mode*]
+ (if (= :web (:source mode))
+ (json/decode-from-str (:body (http/get (str *comments-api* ns "/"
+ (fixup-name-url name)))))
+ (let [name-info (get (:data mode) (str ns "/" name))]
+ (:comments name-info)))))
(defmacro comments
@@ -170,9 +222,13 @@
(defn see-also-core
"Return 'see also' info from clojuredocs for a given namespace and name (as strings)"
- ([ns name]
+ [ns name]
+ (let [mode @*cd-client-mode*]
+ (if (= :web (:source mode))
(json/decode-from-str (:body (http/get (str *seealso-api* ns "/"
- (fixup-name-url name)))))))
+ (fixup-name-url name)))))
+ (let [name-info (get (:data mode) (str ns "/" name))]
+ (:see-alsos name-info)))))
(defmacro see-also
@@ -186,7 +242,7 @@
(defn browse-to-core
"Open a browser to the clojuredocs page for a given namespace and name (as strings)"
([ns name]
- (when-let [url (:url (examples ns name))]
+ (when-let [url (:url (examples-core ns name))]
(browse-url url))))
@@ -196,3 +252,62 @@
`(handle-fns-etc ~name browse-to-core))
([ns name]
`(browse-to-core ~ns ~name)))
+
+
+;; Collect lots of info about each name:
+;; + examples, see also list, and comments
+;; + DON'T get the Clojure documentation string. Assume we have that
+;; locally already.
+;;
+;; Use search-str "let" to get a partial snapshot, with only those
+;; names that contain "let". This currently returns 39 results, so it
+;; is a nice smaller test case for development and debugging.
+;;
+;; Use search-str "" to get a full snapshot. As of Mar 3, 2011 that
+;; is information on a little over 4000 names, requiring 3 API calls
+;; per name. Best to ask permission before hitting the server with
+;; this kind of use.
+
+(defn make-snapshot [search-str fname & quiet]
+ (let [verbose (not quiet)
+ all-names-urls (search search-str)
+ junk (when verbose
+ (println "Retrieved basic information for" (count all-names-urls)
+ "names. Getting full details..."))
+ all-info (doall
+ (map (fn [{ns :ns, name :name, :as m}]
+ ;; Make each of ex, sa, and com always a
+ ;; vector, never nil. If examples returns
+ ;; non-nil, it includes both a vector of
+ ;; examples and a URL. We discard the URL
+ ;; here, since it is already available in
+ ;; all-names-urls.
+ (let [junk (when verbose
+ (print (str ns "/" name) " examples:")
+ (flush))
+ ex (if-let [x (examples ns name)]
+ (:examples x)
+ [])
+ junk (when verbose
+ (print (count ex) " see-alsos:")
+ (flush))
+ sa (if-let [x (see-also ns name)] x [])
+ junk (when verbose
+ (print (count sa) " comments:")
+ (flush))
+ com (if-let [x (comments ns name)] x [])
+ junk (when verbose
+ (println (count com)))]
+ (assoc m :examples ex :see-alsos sa :comments com)))
+ all-names-urls))
+ all-info-map (reduce (fn [big-map one-name-info]
+ (assoc big-map
+ (str (:ns one-name-info) "/"
+ (:name one-name-info))
+ one-name-info))
+ {} all-info)]
+ (with-open [f (java.io.OutputStreamWriter.
+ (java.io.BufferedOutputStream.
+ (java.io.FileOutputStream. fname)))]
+ (binding [*out* f]
+ (pprint all-info-map)))))
Oops, something went wrong.

0 comments on commit a2d5e4a

Please sign in to comment.