Skip to content
Browse files

Merge branch 'release/0.92.3'

  • Loading branch information...
2 parents 2fa57e9 + f7198f5 commit 585b4cfead28a9a20f1a14a51f93f99321486236 @davidsantiago committed Nov 17, 2013
Showing with 203 additions and 24 deletions.
  1. +1 −0 .gitignore
  2. +6 −0 README.markdown
  3. +1 −1 project.clj
  4. +71 −22 src/clojure_hbase/core.clj
  5. +124 −1 test/clojure_hbase/core_test.clj
View
1 .gitignore
@@ -7,4 +7,5 @@ classes
pom.xml
.lein-deps-sum
.lein-failures
+.lein-repl-history
target
View
6 README.markdown
@@ -164,6 +164,8 @@ version I've tested against, but earlier ones may still work.
## Lately...
+- Update to version 0.92.3. Tested against 0.92 and 0.94 HBase. Adds an :all-versions variant to the delete function, which deletes all versions of a cell, patch by [Lin Zhemin](https://github.com/miaoski). Also fixes a bug in the :with-timestamp and :with-timestamp-before variants of delete, which parsed the arguments incorrectly. Finally, adds a :with-timestamp variant of put, which allows the insertion of an element with a specific timestamp. Unit tests expanded.
+
- Update to version 0.92.2. Tested against 0.92 and 0.94 HBase. Updates library to work properly with Clojure 1.5. Add check-and-delete operation, patch by [Kris Jenkins](https://github.com/krisajenkins). to-bytes is deprecated as a public function. Deprecate `modify` and `query` in favor of `execute`, which they are both now implemented in terms of. Other bug fixes and unit tests added.
- Update to version 0.92.1. Tested against 0.92 and 0.94 HBase. Updates type hints to the more general HTableInterface instead of the older HTable concrete type. Patch by [Ryan Senior](https://github.com/senior).
@@ -202,6 +204,10 @@ Added a first cut at most of the Admin functions.
I love receiving pull requests and working with people on patches. Please keep in mind, though, that writing functions to wrap HBase's API is usually pretty easy. Most of the work these days is in testing and verification. It really helps me out and speeds things along if you can include unit tests, or at least snippets of repl code you used to verify functionality, that can be turned into unit tests.
+### Testing
+
+The tests run against a mock, embedded version of HBase. You should stop any local installation of HBase, if any is running. If you use see errors saying "Region is not online", ignore it. It should be [HBASE-9303](https://issues.apache.org/jira/browse/HBASE-9303).
+
## License
Eclipse Public License
View
2 project.clj
@@ -1,4 +1,4 @@
-(defproject clojure-hbase "0.92.2"
+(defproject clojure-hbase "0.92.3"
:description "A convenient Clojure interface to HBase."
:dependencies [[org.clojure/clojure "1.5.1"]
[org.apache.hbase/hbase "0.92.2"]
View
93 src/clojure_hbase/core.clj
@@ -416,6 +416,8 @@
(def ^{:private true} put-argnums
{:value 1 ;; :value [:family :column <value>]
:values 1 ;; :values [:family [:column1 value1 ...] ...]
+ :with-timestamp 2 ;; :with-timestamp ts [:value [:family ...] ...],
+ ;; any number of :value or :values nested
:write-to-WAL 1 ;; :write-to-WAL true/false
:row-lock 1 ;; :row-lock <a row lock you've got>
:use-existing 1}) ;; :use-existing <a Put you've made>
@@ -439,38 +441,68 @@
(new Put row))]
[put-obj (filter #(not (contains? directives (first %))) options)]))
+;; Separate functions for -ts variants, despite code duplication, are
+;; to call the functions in a way that lets the server determine the
+;; timestamp when there is none specified. Seems the less surprising
+;; behavior.
+
(defn- put-add
- [#^Put put-op family qualifier value]
+ [^Put put-op family qualifier value]
(.add put-op (to-bytes family) (to-bytes qualifier) (to-bytes value)))
+(defn- put-add-ts
+ [^Put put-op ts family qualifier value]
+ (.add put-op (to-bytes family) (to-bytes qualifier) ts (to-bytes value)))
+
(defn- handle-put-values
- [#^Put put-op values]
+ [^Put put-op values]
(doseq [value (partition 2 values)]
(let [[family qv-pairs] value]
(doseq [[q v] (partition 2 qv-pairs)]
(put-add put-op family q v))))
put-op)
+(defn- handle-put-values-ts
+ [^Put put-op ts values]
+ (doseq [value (partition 2 values)]
+ (let [[family qv-pairs] value]
+ (doseq [[q v] (partition 2 qv-pairs)]
+ (put-add-ts put-op ts family q v))))
+ put-op)
+
+(defn- handle-put-with-ts
+ "Expects spec to be of the form [:with-timestamp ts [...]], where the ellipses
+ are any number of :value or :values specs."
+ [^Put put-op spec]
+ (let [[_ ts sub-specs] spec
+ sub-specs (partition-query sub-specs put-argnums)]
+ (doseq [sub-spec sub-specs]
+ (condp = (first sub-spec)
+ :value (apply put-add-ts put-op ts (second sub-spec))
+ :values (apply handle-put-values-ts put-op ts (second sub-spec))))
+ put-op))
+
(defn put*
"Returns a Put object suitable for performing a put on an HTable. To make
modifications to an existing Put object, pass it as the argument to
:use-existing; to use an existing RowLock, pass it as the argument to
:row-lock."
[row & args]
(let [specs (partition-query args put-argnums)
- [#^Put put-op specs] (make-put row specs)]
+ [^Put put-op specs] (make-put row specs)]
(doseq [spec specs]
(condp = (first spec)
:value (apply put-add put-op (second spec))
:values (handle-put-values put-op (second spec))
+ :with-timestamp (handle-put-with-ts put-op spec)
:write-to-WAL (.setWriteToWAL put-op (second spec))))
put-op))
(defn put
"Creates and executes a Put object against the given table. Options are
the same as for put."
- [#^HTableInterface table row & args]
- (let [p #^Put (apply put* row args)]
+ [^HTableInterface table row & args]
+ (let [p ^Put (apply put* row args)]
(io!
(.put table p))))
@@ -516,8 +548,10 @@
:columns 1 ;; :columns [:family-name [:q1 :q2...]...]
:family 1 ;; :family :family-name
:families 1 ;; :families [:family1 :family2 ...]
- :with-timestamp 2 ;; :with-timestamp <long> [:column [...]
- :with-timestamp-before 2 ;; :with-timestamp-before <long> [:column ...]
+ :with-timestamp 2 ;; :with-timestamp <long> [:column [...] ...]
+ :with-timestamp-before 2 ;; :with-timestamp-before <long> [:column [...] ...]
+ :all-versions 1 ;; :all-versions [:column [:cf :cq]
+ ;; :columns [:cf [...]] ...]
:row-lock 1 ;; :row-lock <a row lock you've got>
:use-existing 1}) ;; :use-existing <a Put you've made>
@@ -542,52 +576,66 @@
[delete-obj (filter #(not (contains? directives (first %))) options)]))
(defn- delete-column
- [#^Delete delete-op family qualifier]
+ [^Delete delete-op family qualifier]
(.deleteColumn delete-op (to-bytes family) (to-bytes qualifier)))
+(defn- delete-columns
+ [^Delete delete-op family qualifier]
+ (.deleteColumns delete-op (to-bytes family) (to-bytes qualifier)))
+
(defn- delete-column-with-timestamp
- [#^Delete delete-op family qualifier timestamp]
+ [^Delete delete-op family qualifier timestamp]
(.deleteColumn delete-op (to-bytes family) (to-bytes qualifier) timestamp))
(defn- delete-column-before-timestamp
- [#^Delete delete-op family qualifier timestamp]
+ [^Delete delete-op family qualifier timestamp]
(.deleteColumns delete-op (to-bytes family) (to-bytes qualifier) timestamp))
(defn- delete-family
- [#^Delete delete-op family]
+ [^Delete delete-op family]
(.deleteFamily delete-op (to-bytes family)))
(defn- delete-family-timestamp
- [#^Delete delete-op family timestamp]
+ [^Delete delete-op family timestamp]
(.deleteFamily delete-op (to-bytes family) timestamp))
(defn- handle-delete-ts
- [#^Delete delete-op ts-specs]
+ [^Delete delete-op ts-specs]
(doseq [[ts-op timestamp specs] (partition 3 ts-specs)
- spec specs]
+ spec (partition-query specs delete-argnums)]
(condp = ts-op
- :with-timestamp
+ :with-timestamp
(condp = (first spec)
- :column
+ :column
(apply #(delete-column-with-timestamp delete-op %1 %2 timestamp)
- (rest spec))
- :columns (let [[family quals] (rest spec)]
+ (first (rest spec)))
+ :columns (let [[family quals] (first (rest spec))]
(doseq [q quals]
(delete-column-with-timestamp
- delete-op family q timestamp))))
+ delete-op family q timestamp))))
:with-timestamp-before
(condp = (first spec)
- :column
+ :column
(apply #(delete-column-before-timestamp delete-op %1 %2 timestamp)
- (rest spec))
- :columns (let [[family quals] (rest spec)]
+ (first (rest spec)))
+ :columns (let [[family quals] (first (rest spec))]
(doseq [q quals]
(delete-column-before-timestamp
delete-op family q timestamp)))
:family (delete-family-timestamp delete-op (second spec) timestamp)
:families (doseq [f (rest spec)]
(delete-family-timestamp delete-op f timestamp))))))
+(defn- delete-all-versions
+ [^Delete delete-op specs]
+ (doseq [spec (partition-query specs delete-argnums)]
+ (condp = (first spec)
+ :column
+ (apply #(delete-columns delete-op %1 %2) (first (rest spec)))
+ :columns (let [[family quals] (first (rest spec))]
+ (doseq [q quals]
+ (delete-columns delete-op family q))))))
+
(defn delete*
"Returns a Delete object suitable for performing a delete on an HTable. To
make modifications to an existing Delete object, pass it as the argument to
@@ -600,6 +648,7 @@
(condp = (first spec)
:with-timestamp (handle-delete-ts delete-op spec)
:with-timestamp-before (handle-delete-ts delete-op spec)
+ :all-versions (delete-all-versions delete-op (second spec))
:column (apply #(delete-column delete-op %1 %2)
(second spec))
:columns (apply-columns #(delete-column delete-op %1 %2)
View
125 test/clojure_hbase/core_test.clj
@@ -21,6 +21,11 @@
(as-vector result :map-family keywordize :map-qualifier keywordize
:map-timestamp (fn [x] nil) :map-value keywordize))
+(defn test-vector-ts
+ [result]
+ (as-vector result :map-family keywordize :map-qualifier keywordize
+ :map-value keywordize))
+
(defn test-map
[result]
(latest-as-map result :map-family keywordize :map-qualifier keywordize
@@ -138,6 +143,36 @@
[cf-name :testqual])))
"Successfully executed Delete of the Put.")))))
+(deftest put-timestamped
+ (let [cf-name "test-cf-name"
+ row "testrow"
+ timestamp 1337
+ rowvalue [[:test-cf-name :testqual timestamp :testval]]]
+ (as-test
+ (disable-table test-tbl-name)
+ (add-column-family test-tbl-name (column-descriptor cf-name))
+ (enable-table test-tbl-name)
+ (with-table [test-tbl (table test-tbl-name)]
+ (put test-tbl row :with-timestamp timestamp
+ [:value [cf-name :testqual :testval]])
+ (is (= rowvalue
+ (test-vector-ts
+ (get test-tbl row :column
+ [cf-name :testqual])))
+ "Successfully executed Put :value with timestamp and Get :column.")
+ (is (= rowvalue (test-vector-ts (get test-tbl row
+ :time-stamp timestamp)))
+ "Sucessfully executed Get :time-stamp")
+ (is (= rowvalue (test-vector-ts
+ (get test-tbl row
+ :time-range [(dec timestamp) (inc timestamp)])))
+ "Successfully executed Get :time-range")
+ ;; Delete the row
+ (delete test-tbl row :column [cf-name :testqual])
+ (is (= '() (as-vector (get test-tbl row :column
+ [cf-name :testqual])))
+ "Successfully executed Delete of the Put.")))))
+
(deftest construction-options-test
(let [cf-name "test-cf-name"
row "testrow"
@@ -223,6 +258,94 @@
:test-cf-name2 [:test2qual1 :test2qual2]])))
"Successfully executed Delete of multiple cols using :columns.")))))
+(deftest all-versions-delete
+ (let [row "testrow"
+ rowvalue [[:cf1 :a nil :v1t3]
+ [:cf1 :b nil :v2t1]
+ [:cf2 :c nil :v3t2]
+ [:cf2 :d nil :v4t3]
+ [:cf2 :e nil :v5t1]]
+ deletev1 [[:cf1 :b nil :v2t1]]
+ deletev2 [[:cf2 :e nil :v5t1]]
+ deletev3 [[:cf1 :a nil :final]]]
+ (as-test
+ (disable-table test-tbl-name)
+ (add-column-family test-tbl-name (column-descriptor "cf1"))
+ (add-column-family test-tbl-name (column-descriptor "cf2"))
+ (enable-table test-tbl-name)
+ (with-table [test-tbl (table test-tbl-name)]
+ (put test-tbl row :values [:cf1 [:a "v1t1"
+ :a "v1t2"
+ :a "v1t3"
+ :b "v2t1"]
+ :cf2 [:c "v3t1"
+ :c "v3t2"
+ :d "v4t1"
+ :d "v4t2"
+ :d "v4t3"
+ :e "v5t1"]])
+ (is (= rowvalue (test-vector (get test-tbl row)))
+ "Verified all columns were Put with an unqualified row Get.")
+ (delete test-tbl row :all-versions [:column [:cf1 :a]])
+ (is (= deletev1
+ (test-vector (get test-tbl row :family :cf1)))
+ "Tested :all-versions [:column [cf cq]].")
+ (delete test-tbl row :all-versions [:column [:cf1 :b]
+ :columns [:cf2 [:c :d]]])
+ (is (= deletev2
+ (test-vector (get test-tbl row :family :cf2)))
+ "Tested :all-versions [:columns [cf [cq ...]]] (1/2)")
+ (is (empty?
+ (test-vector (get test-tbl row :family :cf1)))
+ "Tested :all-versions [:columns [cf [cq ...]]] (2/2)")
+ (put test-tbl row :values [:cf1 [:a "final"]])
+ (is (= deletev3
+ (test-vector (get test-tbl row :family :cf1)))
+ "Tested to put a new version after deleting all-versions.")))))
+
+(deftest with-timestamp-delete
+ (let [row "testrow"
+ rowvalue [[:cf1 :a 1 :v1]
+ [:cf1 :b 2 :v2]
+ [:cf1 :c 3 :v3]
+ [:cf2 :d 4 :v4]
+ [:cf2 :e 5 :v5]
+ [:cf2 :f 6 :v6]]
+ deleted-t4 [[:cf1 :a 1 :v1]
+ [:cf1 :b 2 :v2]
+ [:cf1 :c 3 :v3]
+ [:cf2 :e 5 :v5]
+ [:cf2 :f 6 :v6]]
+ deleted-before-t4 [[:cf1 :a 1 :v1]
+ [:cf2 :e 5 :v5]
+ [:cf2 :f 6 :v6]]]
+ (as-test
+ (disable-table test-tbl-name)
+ (add-column-family test-tbl-name (column-descriptor "cf1"))
+ (add-column-family test-tbl-name (column-descriptor "cf2"))
+ (enable-table test-tbl-name)
+ (with-table [test-tbl (table test-tbl-name)]
+ (put test-tbl row
+ :with-timestamp 1 [:value [:cf1 :a :v1]]
+ :with-timestamp 2 [:value [:cf1 :b :v2]]
+ :with-timestamp 3 [:value [:cf1 :c :v3]]
+ :with-timestamp 4 [:value [:cf2 :d :v4]]
+ :with-timestamp 5 [:value [:cf2 :e :v5]]
+ :with-timestamp 6 [:value [:cf2 :f :v6]])
+ (is (= rowvalue (test-vector-ts (get test-tbl row)))
+ "Verified all columns were Put with an unqualified row Get.")
+ (delete test-tbl row :with-timestamp 4 [:column [:cf2 :d]
+ :columns [:cf1 [:a :b]]])
+ (is (= deleted-t4
+ (test-vector-ts (get test-tbl row)))
+ "Tested delete :with-timestamp.")
+ (delete test-tbl row :with-timestamp-before 4
+ [:columns [:cf1 [:b :c]]
+ :column [:cf2 :e]])
+ (is (= deleted-before-t4
+ (test-vector-ts (get test-tbl row :families [:cf1 :cf2])))
+ "Tested delete :with-timestamp-before.")))))
+
(deftest atomic-ops-test
(let [cf-name "test-cf-name"]
(as-test
@@ -436,4 +559,4 @@
(f)
(once-stop))
-(use-fixtures :once once-fixture)
+(use-fixtures :once once-fixture)

0 comments on commit 585b4cf

Please sign in to comment.
Something went wrong with that request. Please try again.