Skip to content
Browse files

Merge branch 'search'

  • Loading branch information...
2 parents 1ffd504 + 5573e79 commit 6daf8cdaa14f596f0c3e846f615bba14984e0892 @michaelklishin committed Sep 12, 2012
View
4 .travis.yml
@@ -1,9 +1,13 @@
language: clojure
lein: lein2
+before_script: lein2 with-profile dev javac
script: lein2 all test
branches:
only:
- master
+ - 1.2.x-stable
+ - 1.1.x-stable
+ - 1.0.x-stable
jdk:
- openjdk6
- openjdk7
View
29 ChangeLog.md
@@ -1,5 +1,34 @@
## Changes between Welle 1.2.0 and 1.3.0
+### Search Support
+
+`clojurewerkz.welle.solr` is a new namespace that provides support for [Riak Search](http://wiki.basho.com/Riak-Search.html) using the Solr API.
+Both indexing and querying are supported:
+
+``` clojure
+(require '[clojurewerkz.welle.solr :as wsolr])
+
+;; indexing
+(wsolr/delete-via-query "an-index" "text:*")
+(wsolr/index bucket-name {:username "clojurewerkz"
+ :text "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure"
+ :timestamp "20120802T101232+0100"
+ :id 1})
+```
+
+``` clojure
+(require '[clojurewerkz.welle.solr :as wsolr])
+
+;; querying
+(let [result (wsolr/search "an-index" "title:feature")
+ hits (wsolr/hits-from result)]
+ (println result)
+ (println hits)
+ (is (> (count hits) 0)))
+```
+
+
+
### Support for SMILE
Welle now provides transparent serialization and deserialization support for SMILE, just like it has for
View
2 bin/ci/before_script.sh
@@ -0,0 +1,2 @@
+# this assumes Riak Search is enabled
+search-cmd set-schema tweets test/resources/search/tweets/schema.erl
View
10 project.clj
@@ -7,8 +7,14 @@
[com.basho.riak/riak-client "1.0.5"]
[cheshire "4.0.2"]
[clojurewerkz/support "0.7.0"]
- [com.novemberain/validateur "1.2.0"]]
- :source-paths ["src/clojure"]
+ [com.novemberain/validateur "1.2.0"]
+ ;; for the Riak Search Solr API support. When Riak Client supports
+ ;; search natively, we should be able to just use what it provides.
+ [clj-http "0.5.2"]
+ [org.clojure/data.xml "0.0.6" :exclusions [org.clojure/clojure]]]
+ :source-paths ["src/clojure"]
+ :java-source-paths ["src/java"]
+ :javac-options ["-target" "1.6" "-source" "1.6"]
:profiles {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]}
:1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]}
:dev {:resource-paths ["test/resources"]
View
18 src/clojure/clojurewerkz/welle/core.clj
@@ -1,8 +1,9 @@
(ns clojurewerkz.welle.core
(:import com.basho.riak.client.raw.RawClient
- [com.basho.riak.client.raw.http HTTPClientAdapter HTTPClusterClient HTTPClusterConfig]
+ [com.basho.riak.client.raw.http HTTPClusterClient HTTPClusterConfig]
[com.basho.riak.client.raw.pbc PBClientAdapter PBClusterClient PBClusterConfig]
- com.basho.riak.client.raw.config.ClusterConfig))
+ com.basho.riak.client.raw.config.ClusterConfig
+ clojurewerkz.welle.HTTPClient))
@@ -20,16 +21,16 @@
(def ^{:const true} default-cluster-connection-limit 32)
-(defn ^com.basho.riak.client.raw.RawClient
+(defn ^clojurewerkz.welle.HTTPClient
connect
([]
(connect default-url))
([^String url]
- (let [c (HTTPClientAdapter. (com.basho.riak.client.http.RiakClient. ^String url))]
+ (let [c (HTTPClient. (com.basho.riak.client.http.RiakClient. ^String url))]
(.generateAndSetClientId c)
c))
([^String url ^bytes client-id]
- (let [^RawClient c (connect url)]
+ (let [^HTTPClient c (connect url)]
(.setClientId c client-id)
c)))
@@ -114,3 +115,10 @@
(defn stats
[]
(.stats *riak-client*))
+
+(defn get-base-url
+ "Returns base HTTP transport URL (e.g. http://127.0.0.1:8098)"
+ ([]
+ (.getBaseUrl ^HTTPClient *riak-client*))
+ ([^HTTPClient client]
+ (.getBaseUrl client)))
View
124 src/clojure/clojurewerkz/welle/solr.clj
@@ -0,0 +1,124 @@
+(ns ^{:doc "Provides access to Riak Search via the Solr API.
+
+ Only HTTP transport is supported."}
+ clojurewerkz.welle.solr
+ (:require [clojurewerkz.welle.core :as wc]
+ [clj-http.client :as http]
+ [cheshire.core :as json]
+ [clojure.data.xml :as x])
+ (:import clojurewerkz.welle.HTTPClient))
+
+;;
+;; Implementation
+;;
+
+(defn- get-base-solr-url
+ "Returns base Sorl API URL (e.g. http://127.0.0.1:8098/solr)"
+ ([]
+ (get-base-solr-url wc/*riak-client*))
+ ([^HTTPClient client]
+ (str (.getBaseUrl client) "/solr")))
+
+(defn- get-solr-query-url
+ "Returns Sorl query endpoint URL for the given index (e.g. http://127.0.0.1:8098/solr/production_index/select)"
+ ([^String index]
+ (get-solr-query-url wc/*riak-client* index))
+ ([^HTTPClient client ^String index]
+ (str (get-base-solr-url wc/*riak-client*) "/" index "/select")))
+
+(defn- get-solr-update-url
+ "Returns Sorl update (index, delete, etc) endpoint URL for the given index (e.g. http://127.0.0.1:8098/solr/production_index/update)"
+ ([^String index]
+ (get-solr-update-url wc/*riak-client* index))
+ ([^HTTPClient client ^String index]
+ (str (get-base-solr-url wc/*riak-client*) "/" index "/update")))
+
+(defn- delete-via-query-body
+ [^String query]
+ (x/emit-str
+ (x/element :delete {}
+ (x/element :query {} query))))
+
+(defn- ->xml-field
+ [[k v]]
+ (x/element :field {:name (name k)} (str v)))
+
+(defn- doc->xml-fields
+ [m]
+ (map ->xml-field m))
+
+(defn- doc->xml
+ [m]
+ (x/element :doc {}
+ (doc->xml-fields m)))
+
+(defn- as-vec
+ [xs]
+ (if (sequential? xs)
+ xs
+ [xs]))
+
+(defn- index-document-body
+ [xs]
+ (x/emit-str (x/element :add {}
+ (map doc->xml (as-vec xs)))))
+
+(def ^{:const true}
+ application-xml "application/xml")
+
+;;
+;; API
+;;
+
+(defn delete-via-query
+ ([^String query]
+ (let [url (get-solr-update-url)
+ {:keys [body]} (http/post url {:content-type application-xml :body (delete-via-query-body query)})]
+ nil))
+ ([^String index ^String query]
+ (let [url (get-solr-update-url index)
+ {:keys [body]} (http/post url {:content-type application-xml :body (delete-via-query-body query)})]
+ ;; looks like the response is always empty
+ nil)))
+
+(defn index
+ ([doc]
+ (let [url (get-solr-update-url)
+ {:keys [body]} (http/post url {:content-type application-xml :body (index-document-body doc)})]
+ doc))
+ ([^String idx doc]
+ (let [url (get-solr-update-url idx)
+ {:keys [body]} (http/post url {:content-type application-xml :body (index-document-body doc)})]
+ ;; looks like the response is always empty
+ doc)))
+
+(defn search
+ [^String index ^String query & {:as options}]
+ (let [url (get-solr-query-url index)
+ qp (merge options {"wt" "json" "q" query})
+ {:keys [body]} (http/get url {:query-params qp})]
+ (json/parse-string body true)))
+
+(defn search-across-all-indexes
+ [^String query & {:as options}]
+ (let [url (get-solr-query-url)
+ qp (merge options {"wt" "json" "q" query})
+ {:keys [body]} (http/get url {:query-params qp})]
+ (json/parse-string body true)))
+
+(defn total-hits
+ [response]
+ (get-in response [:response :numFound]))
+
+(defn any-hits?
+ "Returns true if a response has any search hits, false otherwise"
+ [response]
+ (> (total-hits response) 0))
+
+(def no-hits? (complement any-hits?))
+
+(defn hits-from
+ "Returns search hits from a response as a collection. To retrieve hits overview, get the :hits
+ key from the response"
+ [response]
+ (get-in response [:response :docs]))
View
25 src/java/clojurewerkz/welle/HTTPClient.java
@@ -0,0 +1,25 @@
+package clojurewerkz.welle;
+
+import com.basho.riak.client.http.RiakClient;
+import com.basho.riak.client.http.RiakConfig;
+import com.basho.riak.client.raw.http.HTTPClientAdapter;
+
+/**
+ * Just like {@link com.basho.riak.client.raw.http.HTTPClientAdapter} but exposes its configuration
+ */
+public class HTTPClient extends HTTPClientAdapter {
+ protected final RiakConfig config;
+
+ public HTTPClient(RiakClient client) {
+ super(client);
+ this.config = client.getConfig();
+ }
+
+ public RiakConfig getConfig() {
+ return config;
+ }
+
+ public String getBaseUrl() {
+ return config.getBaseUrl();
+ }
+}
View
25 test/clojurewerkz/welle/test/search_test.clj
@@ -0,0 +1,25 @@
+(ns clojurewerkz.welle.test.search-test
+ (:use clojure.test
+ [clojurewerkz.welle.testkit :only [drain]])
+ (:require [clojurewerkz.welle.core :as wc]
+ [clojurewerkz.welle.kv :as kv]
+ [clojurewerkz.welle.buckets :as wb]
+ [clojurewerkz.welle.solr :as wsolr])
+ (:import com.basho.riak.client.http.util.Constants))
+
+(wc/connect!)
+
+(deftest ^{:search true} test-term-query-via-the-solr-api
+ (let [bucket-name "clojurewerkz.welle.solr.tweets"
+ bucket (wb/update bucket-name :last-write-wins true :enable-search true)
+ doc {:username "clojurewerkz"
+ :text "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure"
+ :timestamp "20120802T101232+0100"
+ :id 1}]
+ (drain bucket-name)
+ (kv/store bucket-name "a-key" doc :content-type "application/json")
+ (let [result (wsolr/search bucket-name "username:clojurewerkz")
+ hits (wsolr/hits-from result)]
+ (is (= "a-key" (-> hits first :id)))
+ (is (> (count hits) 0)))
+ (drain bucket-name)))
View
9 test/resources/search/profiles/documents/1.json
@@ -0,0 +1,9 @@
+{
+ "name": "Alyssa P. Hacker",
+ "username": "alyssa",
+ "bio": "I'm an engineer, making awesome things.",
+ "favorites":{
+ "book": "The Moon is a Harsh Mistress",
+ "album": "Magical Mystery Tour"
+ }
+}
View
5 test/resources/search/tweets/documents/1.json
@@ -0,0 +1,5 @@
+{
+ "username": "clojurewerkz",
+ "text": "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure",
+ "timestamp": "20120802T101232+0100"
+}
View
21 test/resources/search/tweets/schema.erl
@@ -0,0 +1,21 @@
+%% an example used in Riak Handbook by Mathias Meyer, modified to fit Elastisch fixtures
+%% (we just take example documents from there)
+{schema, [{version, "1.1"},
+ {n_val, 2},
+ {default_field, "field"},
+ {default_op, "or"},
+ {analyzer_factory, {erlang, text_analyzers, whitespace_analyzer_factory}}],
+ [{field, [{name, "text"},
+ {type, string},
+ {required, true},
+ {analyzer_factory, {erlang, text_analyzers, standard_analyzer_factory}}]},
+ {field, [{name, "timestamp"},
+ {type, date},
+ {required, true},
+ {analyzer_factory, {erlang, text_analyzers, noop_analyzer_factory}}]},
+ {field, [{name, "username"},
+ {type, string},
+ {required, true},
+ {analyzer_factory, {erlang, text_analyzers, noop_analyzer_factory}} ]},
+ {dynamic_field, [{name, "*"}, {skip, true}]}
+ ]}.

0 comments on commit 6daf8cd

Please sign in to comment.
Something went wrong with that request. Please try again.