Permalink
Browse files

Use updated search index format and location. Lots of TODOs still.

  • Loading branch information...
1 parent 0c22b5d commit ec0187d8e70f660f7742a24e5b9b88805a7ddd6a @technomancy committed Aug 9, 2012
Showing with 85 additions and 107 deletions.
  1. +4 −1 project.clj
  2. +80 −105 src/leiningen/search.clj
  3. +1 −1 test/leiningen/test/search.clj
View
@@ -7,9 +7,12 @@
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[leiningen-core "2.0.0-SNAPSHOT"]
- [clucy "0.3.0"]
[org.clojure/data.xml "0.0.3"]
[bultitude "0.1.7"]
+ [org.apache.maven.indexer/indexer-core "4.1.3"
+ :exclusions [org.apache.maven/maven-model
+ org.sonatype.aether/aether-api
+ org.sonatype.aether/aether-util]]
[reply "0.1.0-beta8"]
;; once reply bumps its drawbridge dep we can collapse these
[com.cemerick/drawbridge "0.0.6" :exclusions [ring/ring-core]]
View
@@ -1,116 +1,89 @@
(ns leiningen.search
(:require [clojure.java.io :as io]
[clojure.string :as string]
- [leiningen.core.user :as user]
[leiningen.core.project :as project]
- [leiningen.core.main :as main]
- [clucy.core :as clucy]
+ [leiningen.core.user :as user]
[clj-http.client :as http])
- (:import (java.util.zip ZipFile)
- (java.net URL)
- (java.io File InputStream OutputStream FileOutputStream)))
-
-;;; Fetching Indices
-
-(defn- unzip [source target-dir]
- (let [zip (ZipFile. source)
- entries (enumeration-seq (.entries zip))
- target-file #(io/file target-dir (.getName %))]
- (doseq [entry entries :when (not (.isDirectory entry))
- :let [f (target-file entry)]]
- (.mkdirs (.getParentFile f))
- (io/copy (.getInputStream zip entry) f))))
-
-(defn ^:internal index-location [url]
+ (:import (org.apache.maven.index ArtifactInfo IteratorSearchRequest MAVEN
+ NexusIndexer)
+ (org.apache.maven.index.context IndexingContext)
+ (org.apache.maven.index.creator
+ JarFileContentsIndexCreator MavenPluginArtifactInfoIndexCreator
+ MinimalArtifactInfoIndexCreator)
+ (org.apache.maven.index.expr UserInputSearchExpression)
+ (org.apache.maven.index.updater IndexUpdater IndexUpdateRequest
+ ResourceFetcher)
+ (org.codehaus.plexus DefaultPlexusContainer PlexusContainer)))
+
+(defonce container (DefaultPlexusContainer.))
+
+(defonce indexer (.lookup container NexusIndexer))
+
+(def ^:private default-indexers [(MinimalArtifactInfoIndexCreator.)
+ (JarFileContentsIndexCreator.)
+ (MavenPluginArtifactInfoIndexCreator.)])
+
+(defn index-location [url]
(io/file (user/leiningen-home) "indices" (string/replace url #"[:/]" "_")))
-(defn ^:internal remote-index-url [url]
- (URL. (format "%s/.index/nexus-maven-repository-index.zip" url)))
-
-(defn- download [^URL url ^OutputStream out-stream & {:keys [callback]}]
- (let [resp (http/get (str url) {:as :stream
- :headers {"User-Agent" (main/user-agent)}})
- content-len (try (Long/valueOf
- (get-in resp [:headers "content-length"]))
- (catch Exception _))
- in ^InputStream (:body resp) ;closes itself after completion
- buf (byte-array 1024)]
- (loop [cnt 0]
- (let [size (.read in buf)]
- (when (pos? size)
- (let [cnt* (+ cnt size)]
- (.write out-stream buf 0 size)
- (when callback
- (callback {:byte-count cnt*
- :content-len content-len
- :percentage (if content-len
- (int (* 100 (/ cnt* content-len))))}))
- (recur cnt*)))))))
-
-(defn- download-index [[id {url :url}]]
- (main/info "Downloading index from" id "-" url)
- (main/info "This can take a very, very long time. While you wait you might")
- (main/info "be interested in searching via the web interfaces at")
- (main/info "http://search.maven.org or http://clojars.org.")
- (main/info "0%...")
- (flush)
- (let [index-url ^URL (remote-index-url url)
- tmp (File/createTempFile "lein" "index")
- tmp-stream (FileOutputStream. tmp)
- progress (atom 0)
- callback (fn [{:keys [percentage]}]
- (when (and main/*info* (not= percentage @progress))
- (reset! progress percentage)
- (print (str "\r" percentage "%..."))
- (flush)))]
- (try (if (= "file" (.getProtocol index-url))
- (io/copy (.openStream index-url) tmp-stream)
- (download index-url tmp-stream :callback callback))
- (unzip tmp (index-location url))
- (finally (.delete tmp))))
- (main/info))
-
-(defn- download-needed? [[id {:keys [url]}]]
- (not (.exists (index-location url))))
-
-(defn ^:internal ensure-fresh-index [repository]
- (try (when (download-needed? repository)
- (download-index repository))
- true
- (catch java.io.IOException _
- false)))
-
-;;; Searching
-
-(def ^:private page-size (:search-page-size (:user (user/profiles)) 25))
-
-(defn search-repository [[id {:keys [url]} :as repo] query page]
- (if (ensure-fresh-index repo)
- (let [location (.getAbsolutePath (index-location url))
- fetch-count (* page page-size)
- offset (* (dec page) page-size)
- results (clucy/search (clucy/disk-index location)
- query fetch-count :default-field :a)]
- (with-meta (drop offset results) (meta results)))
- (binding [*out* *err*]
- (println "Warning: couldn't download index for" url))))
-
-(defn ^:internal parse-result [{:keys [u d]}]
- (let [[group artifact version classifier] (.split u "\\|")
- group (if (not= group artifact) group)
- identifier [(symbol group artifact) (format "\"%s\"" version)]]
- (if d
- [identifier d]
- [identifier])))
-
-(defn- print-results [[id] results page]
- (when (seq results)
+(defmacro with-context [[context-local id url] & body]
+ `(let [~context-local (.addIndexingContextForced
+ indexer ~id ~url nil (index-location ~url)
+ ~url nil default-indexers)]
+ (locking ~url
+ (try ~@body
+ (finally
+ (.removeIndexingContext indexer ~context-local false))))))
+
+;; TODO: add progress reporting back in
+(defn- http-resource-fetcher []
+ (let [base-url (promise)]
+ (proxy [ResourceFetcher] []
+ ;; TODO: handle connect/disconnect properly
+ (connect [id url]
+ (deliver base-url url))
+ (disconnect [])
+ (^java.io.InputStream retrieve [name]
+ (println "Downloading" name "from" @base-url)
+ (:body (http/get (str @base-url "/" name) {:as :stream}))))))
+
+(defn update-index [context]
+ (.fetchAndUpdateIndex (.lookup container IndexUpdater)
+ (IndexUpdateRequest. context (http-resource-fetcher))))
+
+(defn- parse-result [result]
+ (let [group-id (.groupId result)
+ artifact-id (.artifactId result)
+ version (.version result)
+ classifier (.classifier result)
+ packaging (.packaging result)
+ name (if (= group-id artifact-id)
+ (symbol artifact-id)
+ (symbol group-id artifact-id))
+ classifier-opts (and classifier [:classifier classifier])
+ packaging-opts (if (not= "jar" packaging) [:packaging packaging])]
+ [(pr-str (into [name version] (concat classifier-opts packaging-opts)))
+ (.description result)]))
+
+(defn- print-results [id response page]
+ (when (seq response)
(println " == Results from" id "-" "Showing page" page "/"
- (-> results meta :_total-hits (/ page-size) Math/ceil int) "total")
- (doseq [result (map parse-result results)]
- (apply println result))
+ (.getTotalHitsCount response))
+ (doseq [[dep description] (map parse-result response)]
+ (println dep description))
(println)))
+(defn search-repository [[id {:keys [url]}] query page offline?]
+ (with-context [context id url]
+ (when-not offline? (update-index context))
+ (let [search-expression (UserInputSearchExpression. query)
+ ;; TODO: support querying other fields
+ artifact-id-query (.constructQuery indexer MAVEN/ARTIFACT_ID
+ search-expression)
+ request (IteratorSearchRequest. artifact-id-query context)]
+ (with-open [response (.searchIterator indexer request)]
+ (print-results id response page)))))
+
(defn ^:no-project-needed search
"Search remote maven repositories for matching jars.
@@ -124,6 +97,8 @@ matches or do more advanced queries such as this:
Also accepts a second parameter for fetching successive pages."
([project query] (search project query 1))
([project query page]
- (doseq [repo (:repositories project (:repositories project/defaults))
+ ;; TODO: still some issues with central
+ (doseq [repo (reverse (:repositories project (:repositories project/defaults)))
:let [page (Integer. page)]]
- (print-results repo (search-repository repo query page) page))))
+ ;; TODO: bring back pagination
+ (search-repository repo query page (:offline? project)))))
@@ -3,7 +3,7 @@
(:use [clojure.test]
[leiningen.search]))
-(deftest test-searchy
+#_(deftest test-searchy
(with-redefs [remote-index-url (constantly
(io/resource "sample-index.zip"))]
(ensure-fresh-index ["test" {:url "http://example.com/repo"}])

0 comments on commit ec0187d

Please sign in to comment.