Permalink
Browse files

Merge remote-tracking branch 'upstream/master'

  • Loading branch information...
2 parents 63fdf6a + 94ec2f6 commit 5c0132f1fc0a59484ad9f5fbfdc3e90e992845a7 @mstump committed Jun 12, 2012
Showing with 110 additions and 28 deletions.
  1. +3 −3 README.md
  2. +2 −2 project.clj
  3. +25 −8 src/clj_hector/core.clj
  4. +30 −7 src/clj_hector/ddl.clj
  5. +6 −6 src/clj_hector/serialize.clj
  6. +21 −2 test/clj_hector/test/core.clj
  7. +23 −0 test/clj_hector/test/ddl.clj
View
@@ -1,14 +1,14 @@
# clj-hector
-A simple Clojure client for Cassandra that wraps Hector
+A simple Clojure client for Cassandra that wraps Hector. The 0.2.1 release was built against Clojure 1.4.0.
Current build status: ![Build status](https://secure.travis-ci.org/pingles/clj-hector.png)
## Installation
Add the following to your `project.clj`
- :dev-dependencies [[org.clojars.paul/clj-hector "0.1.3"]]
+ :dev-dependencies [[org.clojars.paul/clj-hector "0.2.1"]]
## Usage
@@ -64,7 +64,7 @@ Firstly, the column family will need to support super columns.
Storing super columns works using a nested map structure:
- user> (put ks "UserRelationships" "paul" {"SuperCol" {"k" "v"} "SuperCol2" {"k2" "v2"}})
+ user> (put ks "UserRelationships" "paul" {"SuperCol" {"k" "v"} "SuperCol2" {"k2" "v2"}} :type :super)
#<MutationResultImpl MutationResult took (6us) for query (n/a) on host: localhost(127.0.0.1):9160>
Retrieving super columns with `get-super-rows`:
View
@@ -1,9 +1,9 @@
-(defproject org.clojars.paul/clj-hector "0.1.3"
+(defproject org.clojars.paul/clj-hector "0.2.1"
:description "Wrapper for Hector Cassandra client"
:resources-path "test/resources"
:dev-dependencies [[org.slf4j/slf4j-simple "1.6.3"]
[org.apache.cassandra/cassandra-all "1.0.5"]]
:dependencies [[me.prettyprint/hector-core "1.0-3" :exclusions [org.slf4j/slf4j-log4j12]]
[org.slf4j/slf4j-api "1.6.1"]
- [org.clojure/clojure "1.2.1"]
+ [org.clojure/clojure "1.4.0"]
[joda-time/joda-time "2.0"]])
View
@@ -151,20 +151,37 @@
(HFactory/createColumn name value (int ttl) n-serializer v-serializer)
(HFactory/createColumn name value n-serializer v-serializer))))))
-(defn put
- "Stores values in columns in map m against row key pk"
- [ks cf pk m & opts]
+(defn batch-put
+ "Add multiple rows before executing the put operation. Rows are expressed as
+ a map, i.e. {<row-pk> {<col-k> <col-v>, ... }, <row-pk> {...}, ...}
+
+ NOTE: You will need to experiment to find the right batch size for your
+ specific use case. While a larger batch may improve performance, overly
+ large batches are discouraged. When a batch mutation fails, the entire
+ request must be retried. Additionally, loading very large batches into
+ memory can cause problems on the server."
+ [ks cf rows & {:as opts}]
(let [^Mutator mut (HFactory/createMutator ks type-inferring)
defaults (merge {:n-serializer :type-inferring
:v-serializer :type-inferring
:s-serializer :type-inferring}
- (apply hash-map opts))
- opts (extract-options (apply concat (seq defaults)) cf)]
- (if (counter? opts)
- (doseq [[n v] m] (.addCounter mut pk cf (create-column n v opts)))
- (doseq [[k v] m] (.addInsertion mut pk cf (create-column k v opts))))
+ opts)
+ opts (extract-options (apply concat (seq defaults)) cf)
+ mut-add (if (counter? opts)
+ (fn [pk cf n v]
+ (.addCounter mut pk cf (create-column n v opts)))
+ (fn [pk cf k v]
+ (.addInsertion mut pk cf (create-column k v opts))))]
+ (doseq [[pk col-map] rows
+ [k v] col-map]
+ (mut-add pk cf k v))
(.execute mut)))
+(defn put
+ "Stores values in columns in map m against row key pk"
+ [ks cf pk col-map & opts]
+ (apply batch-put ks cf {pk col-map} opts))
+
(defn create-counter-column
[name value & {:keys [n-serializer v-serializer s-serializer]
:or {n-serializer type-inferring
View
@@ -5,7 +5,10 @@
[me.prettyprint.hector.api Cluster]
[me.prettyprint.cassandra.service ThriftCfDef]
[me.prettyprint.hector.api.beans Composite AbstractComposite$ComponentEquality]
- [me.prettyprint.hector.api.ddl ComparatorType ColumnFamilyDefinition ColumnType KeyspaceDefinition]))
+ [me.prettyprint.hector.api.ddl ComparatorType ColumnFamilyDefinition ColumnType KeyspaceDefinition ColumnIndexType]
+ [me.prettyprint.cassandra.model BasicColumnDefinition BasicColumnFamilyDefinition]
+ [me.prettyprint.cassandra.serializers StringSerializer]))
+
(def component-equality-type {:less_than_equal AbstractComposite$ComponentEquality/LESS_THAN_EQUAL
:equal AbstractComposite$ComponentEquality/EQUAL
@@ -37,6 +40,8 @@
:uuid "UUIDType"
:counter "CounterColumnType"})
+(def column-index-types {:keys ColumnIndexType/KEYS})
+
(defn- column-type
[type]
(if (= :super type)
@@ -47,18 +52,36 @@
[validator]
(get validator-types (or validator :bytes)))
+(defn- make-column
+ "Returns an object defining a column with validation and optional index"
+ ([{:keys [name index-name index-type validator]}]
+ (let [c-def (BasicColumnDefinition.)
+ name (.toByteBuffer (StringSerializer/get) name)]
+ (doto c-def
+ (.setName name)
+ (.setValidationClass (.getClassName (comparator-types validator))))
+ (if index-type
+ (.setIndexType c-def (column-index-types index-type)))
+ (if index-name
+ (.setIndexName c-def index-name))
+ c-def)))
+
(defn- make-column-family
"Returns an object defining a new column family"
- ([keyspace {:keys [name type comparator comparator-alias validator]}]
- (let [cf-def (HFactory/createColumnFamilyDefinition keyspace name)]
- (doto ^ThriftCfDef cf-def
- (.setColumnType (column-type type))
- (.setDefaultValidationClass (default-validation-class validator)))
+ ([keyspace {:keys [name type comparator comparator-alias validator column-metadata]}]
+ (let [cf-def (BasicColumnFamilyDefinition.)
+ columns (map make-column column-metadata)]
+ (doto ^BasicColumnFamilyDefinition cf-def
+ (.setName name)
+ (.setKeyspaceName keyspace)
+ (.setColumnType (column-type type))
+ (.setDefaultValidationClass (default-validation-class validator)))
(if comparator
(.setComparatorType cf-def (comparator-types comparator)))
(if comparator-alias
(.setComparatorTypeAlias cf-def comparator-alias))
- cf-def)))
+ (doseq [column columns] (.addColumnDefinition cf-def column))
+ (ThriftCfDef. cf-def))))
(defn- make-keyspace-definition
([keyspace strategy-class replication-factor column-families]
@@ -53,11 +53,11 @@
implementation in Hector. this isn't neccessary for
DynamicComposite."
[composite deserializers]
- (into []
- (map (fn [component deserializer]
- (.fromByteBuffer (serializer deserializer) (.getBytes component)))
- (.getComponents composite)
- deserializers)))
+ (vec
+ (map (fn [component deserializer]
+ (.fromByteBuffer (serializer deserializer) (.getBytes component)))
+ (.getComponents composite)
+ deserializers)))
(defprotocol ToClojure
@@ -136,7 +136,7 @@
(let [serializers (:c-serializer opts)]
(if serializers
(deserialize-composite s serializers)
- (into [] (map #(.getValue %1) (.getComponents s))))))
+ (vec (map #(.getValue %1) (.getComponents s))))))
QueryResultImpl
(to-clojure [s opts]
(with-meta
@@ -224,7 +224,7 @@
(deftest ttl-columns
(testing "regular column ttl"
(let [column-family "A"
- opts [:n-serializer :string :v-serializer :integer]
+ opts [:n-serializer :string :v-serializer :long]
pk "row-key1"]
(with-test-keyspace keyspace [{:name column-family}]
(put keyspace column-family pk {"n" 1 "n2" 2} :ttl 1)
@@ -235,7 +235,7 @@
(apply get-columns keyspace column-family pk ["n" "n2"] opts))))))
(testing "super column ttl"
(let [column-family "A"
- opts [:s-serializer :string :n-serializer :string :v-serializer :integer]
+ opts [:s-serializer :string :n-serializer :string :v-serializer :long]
pk "row-key"]
(with-test-keyspace keyspace [{:name column-family
:type :super}]
@@ -245,3 +245,22 @@
(Thread/sleep 2000)
(is (= {}
(apply get-super-columns keyspace column-family pk "SuperCol" ["n" "n2"] opts)))))))
+
+(deftest test-batch-put
+ (let [column-family "A"
+ opts [:s-serializer :string :n-serializer :string :v-serializer :long]
+ pk1 "row-key1"
+ pk2 "row-key2"]
+ (with-test-keyspace keyspace [{:name column-family
+ :type :super}]
+ (batch-put keyspace column-family
+ {pk1 {"SuperCol" {"n" 1 "n2" 2}}
+ pk2 {"SuperCol" {"n3" 3 "n4" 4}}}
+ :ttl 1 :type :super)
+ (is (= {"n" 1 "n2" 2}
+ (apply get-super-columns keyspace column-family pk1
+ "SuperCol" ["n" "n2"] opts)))
+ (is (= {"n3" 3 "n4" 4}
+ (apply get-super-columns keyspace column-family pk2
+ "SuperCol" ["n3" "n4"] opts))))))
+
@@ -96,3 +96,26 @@
:validator :counter}
(first (column-families cluster random-ks))))
(drop-keyspace cluster random-ks))))
+
+(deftest should-add-remove-column-families-with-column-meta-data
+ (let [random-ks (.replace (str "ks" (java.util.UUID/randomUUID)) "-" "")]
+ (with-test-cluster cluster
+ (add-keyspace cluster
+ {:name random-ks
+ :strategy :simple
+ :replication 1
+ :column-families [{:name "a"
+ :type :standard
+ :column-metadata
+ [{:name "col"
+ :index-name "colidx"
+ :index-type :keys
+ :validator :utf-8}
+ {:name "coltwo"
+ :validator :integer}]}]})
+ (is (= {:name "a"
+ :comparator :bytes
+ :type :standard
+ :validator :bytes}
+ (first (column-families cluster random-ks))))
+ (drop-keyspace cluster random-ks))))

0 comments on commit 5c0132f

Please sign in to comment.