Merge branch 'search'

.travis.yml

@@ -1,9 +1,13 @@
 language: clojure
 lein: lein2
+before_script: lein2 with-profile dev javac
 script: lein2 all test
 branches:
   only:
     - master
+    - 1.2.x-stable
+    - 1.1.x-stable
+    - 1.0.x-stable
 jdk:
   - openjdk6
   - openjdk7

ChangeLog.md

@@ -1,5 +1,34 @@
 ## Changes between Welle 1.2.0 and 1.3.0
 
+### Search Support
+
+`clojurewerkz.welle.solr` is a new namespace that provides support for [Riak Search](http://wiki.basho.com/Riak-Search.html) using the Solr API.
+Both indexing and querying are supported:
+
+``` clojure
+(require '[clojurewerkz.welle.solr    :as wsolr])
+
+;; indexing
+(wsolr/delete-via-query "an-index" "text:*")
+(wsolr/index bucket-name {:username  "clojurewerkz"
+                          :text      "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure"
+                          :timestamp "20120802T101232+0100"
+                          :id        1})
+```
+
+``` clojure
+(require '[clojurewerkz.welle.solr    :as wsolr])
+
+;; querying
+(let [result (wsolr/search "an-index" "title:feature")
+      hits   (wsolr/hits-from result)]
+  (println result)
+  (println hits)
+  (is (> (count hits) 0)))
+```
+
+
+
 ### Support for SMILE 
 
 Welle now provides transparent serialization and deserialization support for SMILE, just like it has for

bin/ci/before_script.sh

@@ -0,0 +1,2 @@
+# this assumes Riak Search is enabled
+search-cmd set-schema tweets test/resources/search/tweets/schema.erl

project.clj

@@ -7,8 +7,14 @@
                  [com.basho.riak/riak-client "1.0.5"]
                  [cheshire                   "4.0.2"]
                  [clojurewerkz/support       "0.7.0"]
-                 [com.novemberain/validateur "1.2.0"]]
-  :source-paths ["src/clojure"]
+                 [com.novemberain/validateur "1.2.0"]
+                 ;; for the Riak Search Solr API support. When Riak Client supports
+                 ;; search natively, we should be able to just use what it provides.
+                 [clj-http                   "0.5.2"]
+                 [org.clojure/data.xml       "0.0.6" :exclusions [org.clojure/clojure]]]
+  :source-paths      ["src/clojure"]
+  :java-source-paths ["src/java"]
+  :javac-options     ["-target" "1.6" "-source" "1.6"]
   :profiles       {:1.3 {:dependencies [[org.clojure/clojure "1.3.0"]]}
                    :1.5 {:dependencies [[org.clojure/clojure "1.5.0-master-SNAPSHOT"]]}
                    :dev {:resource-paths ["test/resources"]

src/clojure/clojurewerkz/welle/core.clj

@@ -1,8 +1,9 @@
 (ns clojurewerkz.welle.core
   (:import com.basho.riak.client.raw.RawClient
-           [com.basho.riak.client.raw.http HTTPClientAdapter HTTPClusterClient HTTPClusterConfig]
+           [com.basho.riak.client.raw.http HTTPClusterClient HTTPClusterConfig]
            [com.basho.riak.client.raw.pbc PBClientAdapter PBClusterClient PBClusterConfig]
-           com.basho.riak.client.raw.config.ClusterConfig))
+           com.basho.riak.client.raw.config.ClusterConfig
+           clojurewerkz.welle.HTTPClient))
 
 
 
@@ -20,16 +21,16 @@
 (def ^{:const true} default-cluster-connection-limit 32)
 
 
-(defn ^com.basho.riak.client.raw.RawClient
+(defn ^clojurewerkz.welle.HTTPClient
   connect
   ([]
      (connect default-url))
   ([^String url]
-     (let [c (HTTPClientAdapter. (com.basho.riak.client.http.RiakClient. ^String url))]
+     (let [c (HTTPClient. (com.basho.riak.client.http.RiakClient. ^String url))]
        (.generateAndSetClientId c)
        c))
   ([^String url ^bytes client-id]
-     (let [^RawClient c (connect url)]
+     (let [^HTTPClient c (connect url)]
        (.setClientId c client-id)
        c)))
 
@@ -114,3 +115,10 @@
 (defn stats
   []
   (.stats *riak-client*))
+
+(defn get-base-url
+  "Returns base HTTP transport URL (e.g. http://127.0.0.1:8098)"
+  ([]
+     (.getBaseUrl ^HTTPClient *riak-client*))
+  ([^HTTPClient client]
+     (.getBaseUrl client)))

src/clojure/clojurewerkz/welle/solr.clj

@@ -0,0 +1,124 @@
+(ns ^{:doc "Provides access to Riak Search via the Solr API.
+
+            Only HTTP transport is supported."}
+  clojurewerkz.welle.solr
+  (:require [clojurewerkz.welle.core :as wc]
+            [clj-http.client         :as http]
+            [cheshire.core           :as json]
+            [clojure.data.xml        :as x])
+  (:import clojurewerkz.welle.HTTPClient))
+
+;;
+;; Implementation
+;;
+
+(defn- get-base-solr-url
+  "Returns base Sorl API URL (e.g. http://127.0.0.1:8098/solr)"
+  ([]
+     (get-base-solr-url wc/*riak-client*))
+  ([^HTTPClient client]
+     (str (.getBaseUrl client) "/solr")))
+
+(defn- get-solr-query-url
+  "Returns Sorl query endpoint URL for the given index (e.g. http://127.0.0.1:8098/solr/production_index/select)"
+  ([^String index]
+     (get-solr-query-url wc/*riak-client* index))
+  ([^HTTPClient client ^String index]
+     (str (get-base-solr-url wc/*riak-client*) "/" index "/select")))
+
+(defn- get-solr-update-url
+  "Returns Sorl update (index, delete, etc) endpoint URL for the given index (e.g. http://127.0.0.1:8098/solr/production_index/update)"
+  ([^String index]
+     (get-solr-update-url wc/*riak-client* index))
+  ([^HTTPClient client ^String index]
+     (str (get-base-solr-url wc/*riak-client*) "/" index "/update")))
+
+(defn- delete-via-query-body
+  [^String query]
+  (x/emit-str
+   (x/element :delete {}
+              (x/element :query {} query))))
+
+(defn- ->xml-field
+  [[k v]]
+  (x/element :field {:name (name k)} (str v)))
+
+(defn- doc->xml-fields
+  [m]
+  (map ->xml-field m))
+
+(defn- doc->xml
+  [m]
+  (x/element :doc {}
+             (doc->xml-fields m)))
+
+(defn- as-vec
+  [xs]
+  (if (sequential? xs)
+    xs
+    [xs]))
+
+(defn- index-document-body
+  [xs]
+  (x/emit-str (x/element :add {}
+                         (map doc->xml (as-vec xs)))))
+
+(def ^{:const true}
+  application-xml "application/xml")
+
+;;
+;; API
+;;
+
+(defn delete-via-query
+  ([^String query]
+     (let [url            (get-solr-update-url)
+           {:keys [body]} (http/post url {:content-type application-xml :body (delete-via-query-body query)})]
+       nil))
+  ([^String index ^String query]
+     (let [url            (get-solr-update-url index)
+           {:keys [body]} (http/post url {:content-type application-xml :body (delete-via-query-body query)})]
+       ;; looks like the response is always empty
+       nil)))
+
+(defn index
+  ([doc]
+     (let [url            (get-solr-update-url)
+           {:keys [body]} (http/post url {:content-type application-xml :body (index-document-body doc)})]
+       doc))
+  ([^String idx doc]
+     (let [url            (get-solr-update-url idx)
+           {:keys [body]} (http/post url {:content-type application-xml :body (index-document-body doc)})]
+       ;; looks like the response is always empty
+       doc)))
+
+(defn search
+  [^String index ^String query & {:as options}]
+  (let [url            (get-solr-query-url index)
+        qp             (merge options {"wt" "json" "q" query})
+        {:keys [body]} (http/get url {:query-params qp})]
+    (json/parse-string body true)))
+
+(defn search-across-all-indexes
+  [^String query & {:as options}]
+  (let [url            (get-solr-query-url)
+        qp             (merge options {"wt" "json" "q" query})
+        {:keys [body]} (http/get url {:query-params qp})]
+    (json/parse-string body true)))
+
+(defn total-hits
+  [response]
+  (get-in response [:response :numFound]))
+
+(defn any-hits?
+  "Returns true if a response has any search hits, false otherwise"
+  [response]
+  (> (total-hits response) 0))
+
+(def no-hits? (complement any-hits?))
+
+(defn hits-from
+  "Returns search hits from a response as a collection. To retrieve hits overview, get the :hits
+   key from the response"
+  [response]
+  (get-in response [:response :docs]))

src/java/clojurewerkz/welle/HTTPClient.java

@@ -0,0 +1,25 @@
+package clojurewerkz.welle;
+
+import com.basho.riak.client.http.RiakClient;
+import com.basho.riak.client.http.RiakConfig;
+import com.basho.riak.client.raw.http.HTTPClientAdapter;
+
+/**
+ * Just like {@link com.basho.riak.client.raw.http.HTTPClientAdapter} but exposes its configuration
+ */
+public class HTTPClient extends HTTPClientAdapter {
+  protected final RiakConfig config;
+
+  public HTTPClient(RiakClient client) {
+    super(client);
+    this.config = client.getConfig();
+  }
+
+  public RiakConfig getConfig() {
+    return config;
+  }
+
+  public String getBaseUrl() {
+    return config.getBaseUrl();
+  }
+}

test/clojurewerkz/welle/test/search_test.clj

@@ -0,0 +1,25 @@
+(ns clojurewerkz.welle.test.search-test
+  (:use clojure.test
+        [clojurewerkz.welle.testkit :only [drain]])
+  (:require [clojurewerkz.welle.core :as wc]
+            [clojurewerkz.welle.kv   :as kv]
+            [clojurewerkz.welle.buckets :as wb]
+            [clojurewerkz.welle.solr    :as wsolr])
+  (:import  com.basho.riak.client.http.util.Constants))
+
+(wc/connect!)
+
+(deftest ^{:search true} test-term-query-via-the-solr-api
+  (let [bucket-name "clojurewerkz.welle.solr.tweets"
+        bucket      (wb/update bucket-name :last-write-wins true :enable-search true)
+        doc         {:username  "clojurewerkz"
+                     :text      "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure"
+                     :timestamp "20120802T101232+0100"
+                     :id        1}]
+    (drain bucket-name)
+    (kv/store bucket-name "a-key" doc :content-type "application/json")
+    (let [result (wsolr/search bucket-name "username:clojurewerkz")
+          hits   (wsolr/hits-from result)]
+      (is (= "a-key" (-> hits first :id)))
+      (is (> (count hits) 0)))
+    (drain bucket-name)))

test/resources/search/profiles/documents/1.json

@@ -0,0 +1,9 @@
+{
+ "name": "Alyssa P. Hacker",
+ "username": "alyssa",
+ "bio": "I'm an engineer, making awesome things.",
+ "favorites":{
+              "book": "The Moon is a Harsh Mistress",
+              "album": "Magical Mystery Tour"
+             }
+}

test/resources/search/tweets/documents/1.json

@@ -0,0 +1,5 @@
+{
+ "username": "clojurewerkz",
+ "text": "Elastisch beta3 is out, several more @elasticsearch features supported github.com/clojurewerkz/elastisch, improved docs http://clojureelasticsearch.info #clojure",
+ "timestamp": "20120802T101232+0100"
+}

test/resources/search/tweets/schema.erl

@@ -0,0 +1,21 @@
+%% an example used in Riak Handbook by Mathias Meyer, modified to fit Elastisch fixtures
+%% (we just take example documents from there)
+{schema, [{version,       "1.1"},
+          {n_val,         2},
+          {default_field, "field"},
+          {default_op,    "or"},
+          {analyzer_factory, {erlang, text_analyzers, whitespace_analyzer_factory}}],
+ [{field, [{name, "text"},
+           {type, string},
+           {required, true},
+           {analyzer_factory, {erlang, text_analyzers, standard_analyzer_factory}}]},
+  {field, [{name, "timestamp"},
+           {type, date},
+           {required, true},
+           {analyzer_factory, {erlang, text_analyzers, noop_analyzer_factory}}]},
+  {field, [{name, "username"},
+           {type, string},
+           {required, true},
+           {analyzer_factory, {erlang, text_analyzers, noop_analyzer_factory}} ]},
+  {dynamic_field, [{name, "*"}, {skip, true}]}
+ ]}.