Permalink
Browse files

added the possibility of specifying encoding/decoding functions when …

…flattening objects to be stored in hbase
  • Loading branch information...
1 parent 2bb2215 commit ca8347b9006f05b2a55d32ae6ecfdbfc7195f87f @amitrathore committed Feb 3, 2009
Showing with 79 additions and 33 deletions.
  1. +13 −2 spec/org/rathore/amit/capjure_spec.clj
  2. +42 −17 spec_suite.clj
  3. +24 −14 src/org/rathore/amit/capjure.clj
@@ -91,5 +91,16 @@
(is (= (consumer "id") "103"))
(is (= (merchant "name") "portable chairs"))))
-(def sku-string "{\"merchant\": {\"id\": 11, \"name\": \"portable chairs\"}, \"active_campaigns\": [3, 4, 7, 10, 11, 13], \"consumer\": {\"kind\": \"visitor\", \"id\": 103, \"email_address\": \"f80a2173-5923-264f-e2d5-cb0f96220220@visitor.cinchcorp.com\"}, \"session\": {\"uber_session_id\": \"a96ec02e-0fd3-b030-c14a-1761ffe7d45b\", \"merchant_session_id\": \"3c0e276524dc843debaebfd9506138ee\", \"cinch_session_id\": \"42e00dc0eae2706a75eee1c1964d4d43\"}, \"api\": \"0.0.1.0\", \"inserts\": [{\"cinch_unit_price\": 36.95, \"campaign_id\": -1, \"html_id\": \"cinch_id_1231729409882\", \"merchant_product_id\": \"SS-REG\", \"sku\": \"SS-REG Black XL\", \"merchant_unit_price\": 36.95, \"insert_type\": \"campaign\"}]}")
-(def sku-object (json/decode-from-str sku-string))
+(def cart-string
+ "{\"event_type\": \"cart\", \"api\": \"0.0.1.0\", \"session\": {\"cinch_session_id\": \"1948c2cef48e58a6c1f215a545512077\", \"merchant_session_id\": \"b768e9911893b91b279e08ac8efde20f\", \"uber_session_id\": \"ee2bcc43-dae9-e0f0-40e8-325f96b5f114\"}, \"cart_items\": [{\"campaign_id\": -1, \"merchant_product_id\": \"EZ-30DC-HD\", \"merchant_unit_price\": 109.95, \"options\": null, \"name\": null, \"cinch_unit_price\": 109.95, \"sku\": \"EZ-30DC-HD (Yellow) XX\", \"quantity\": 1, \"description\": null}, {\"campaign_id\": -1, \"merchant_product_id\": \"OB-BW-LNGR\", \"merchant_unit_price\": 229.95, \"options\": null, \"name\": null, \"cinch_unit_price\": 229.95, \"sku\": \"OB-BW-LNGR (RED) LL\", \"quantity\": 1, \"description\": null}], \"page\": {\"merchant_template_name\": \"index\", \"referrer\": \"http://chairs.vasanta.hq.cinchcorp.com/shopping_cart.html?number_of_uploads=0\", \"request_url\": \"http://chairs.vasanta.hq.cinchcorp.com/hanging-chairs-c-23.html\"}, \"cart\": {\"checkout_state\": 1}, \"merchant\": {\"name\": \"Portable Folding Chairs\", \"id\": 14}, \"http_client\": {\"browser_version\": \"3\", \"operating_system\": \"Macintosh\", \"operating_system_version\": \"OS X\", \"ip_address\": \"192.168.10.10\", \"browser\": \"Safari\"}, \"active_campaigns\": [3, 4, 7, 10, 11, 13], \"consumer\": {\"email_address\": \"afd4cb68-bb3b-e51f-ac2a-cb6eefce528f@visitor.cinchcorp.com\", \"kind\": \"visitor\", \"id\": 103}}")
+(def cart-object (json/decode-from-str cart-string))
+
+(deftest test-flatten-with-sku
+ (let [flattened (flatten cart-object)
+ hydrated (hydrate flattened)
+ cart-items (hydrated "cart_items")
+ cart-item (first cart-items)]
+ (is (= (count cart-items) 2))
+ (is (= (cart-item "sku") "OB-BW-LNGR (RED) LL"))
+ (is (= (cart-item "merchant_product_id") "OB-BW-LNGR"))))
+
View
@@ -1,27 +1,52 @@
(use 'clojure.contrib.test-is)
-(import '(java.net URLEncoder))
+(import '(java.net URLEncoder URLDecoder))
(load-file "spec/org/rathore/amit/capjure_spec.clj")
-
(in-ns 'capjure-spec)
(println "**************** running tests ******************")
-;(def keys-config
-; {:inserts :merchant_product_id
-; :cart_items :merchant_product_id
-; :latest_consumer :merchant_id})
+(defstruct key-config :qualifier :functor)
+(defn add-key-config [keys-config name qualifier functor]
+ (assoc keys-config name (struct key-config qualifier functor)))
+(defn encode-keys []
+ (let [keys-config {}
+ keys-config (add-key-config keys-config
+ :inserts :merchant_product_id
+ (fn [insert-map]
+ (insert-map :merchant_product_id)))
+ keys-config (add-key-config keys-config
+ :latest_consumer :merchant_id
+ (fn [consumer-map]
+ (consumer-map :merchant_id)))
+ keys-config (add-key-config keys-config
+ :cart_items
+ :merchant_product_id (fn [cart-item-map]
+ (let [sku (cart-item-map :sku)
+ m-pid (cart-item-map :merchant_product_id)]
+ (cond
+ (nil? sku) m-pid
+ :else (str m-pid "@" (URLEncoder/encode sku "UTF-8"))))))]
+ keys-config))
+
+(defn decode-keys []
+ (let [keys-config {}
+ keys-config (add-key-config keys-config
+ :inserts :merchant_product_id
+ (fn [value] value))
+ keys-config (add-key-config keys-config
+ :latest_consumer :merchant_id
+ (fn [value] value))
+ keys-config (add-key-config keys-config
+ :cart_items
+ :merchant_product_id (fn [value]
+ (cond
+ (.contains value "@") (first (.split (URLDecoder/decode value "UTF-8") "@"))
+ :else value)))]
+ keys-config))
+
+(def keys-config {:encode (encode-keys) :decode (decode-keys)})
(binding [*hbase-master* "tank.cinchcorp.com:60000"
*primary-keys-config* keys-config]
- (run-tests))
+ (run-tests)
(shutdown-agents))
-
-(def keys-config {
- :inserts (fn [insert-map]
- (insert-map :merchant_product_id))
- :latest_consumer (fn [consumer-map]
- (consumer-map :merchant_id))
- :cart_items (fn [cart-item-map]
- (str (cart-item-map :merchant_product_id) "@" (URLEncoder/encode (cart-item-map :sku))))})
-
-;create struct to track primary key, and function associated with it
@@ -4,11 +4,28 @@
'(org.apache.hadoop.hbase.client HTable Scanner)
'(org.apache.hadoop.hbase.io BatchUpdate Cell))
-
(def *mock-mode* false)
(def *hbase-master* "localhost:60000")
(def *primary-keys-config* {})
+(declare symbol-name)
+(defn encoding-keys []
+ (*primary-keys-config* :encode))
+(defn decoding-keys []
+ (*primary-keys-config* :decode))
+(defn qualifier-for [key-name]
+ (((encoding-keys) key-name) :qualifier))
+(defn encoding-functor-for [key-name]
+ (((encoding-keys) key-name) :functor))
+(defn all-primary-keys []
+ (map #(symbol-name %) (keys (encoding-keys))))
+(defn primary-key [column-family]
+ (first (filter #(.startsWith column-family (str %)) (all-primary-keys))))
+(defn decoding-functor-for [key-name]
+ (((decoding-keys) (keyword key-name)) :functor))
+(defn decode-with-key [key-name value]
+ ((decoding-functor-for key-name) value))
+
(declare flatten add-to-insert-batch capjure-insert hbase-table read-row read-cell)
(defn capjure-insert [object-to-save hbase-table-name row-id]
(let [table (hbase-table hbase-table-name)
@@ -63,10 +80,11 @@
:else (process-strings key (to-array all)))))
(defn process-maps [key maps]
- (let [qualifier (*primary-keys-config* key)]
+ (let [qualifier (qualifier-for key)
+ encoding-functor (encoding-functor-for key)]
(apply merge (map
(fn [single-map]
- (process-map (symbol-name key) (single-map qualifier) (dissoc single-map qualifier)))
+ (process-map (symbol-name key) (encoding-functor single-map) (dissoc single-map qualifier)))
maps))))
(defn process-map [initial-prefix final-prefix single-map]
@@ -87,17 +105,10 @@
(seq bloated_object))))
(declare read-as-hash cell-value-as-string hydrate-pair has-many-strings-hydration has-many-objects-hydration has-one-string-hydration has-one-object-hydration collapse-for-hydration)
-
-(defn all-primary-keys []
- (map #(symbol-name %) (keys *primary-keys-config*)))
-
(defn is-from-primary-keys [key-name]
(let [key-name-str (symbol-name key-name)]
(some #(.startsWith key-name-str %) (all-primary-keys))))
-(defn primary-key [column-family]
- (first (filter #(.startsWith column-family (str %)) (all-primary-keys))))
-
(defn column-name-empty? [key-name]
(= 1 (count (.split key-name ":"))))
@@ -144,9 +155,9 @@
(defn has-many-objects-hydration [hydrated column-family column-name value]
(let [outer-key (primary-key column-family)
inner-key (.substring column-family (+ 1 (count outer-key)) (count column-family))
- primary-key-name (*primary-keys-config* outer-key)
+ primary-key-name (qualifier-for (keyword outer-key))
inner-map (or (hydrated outer-key) {})
- inner-object (or (inner-map column-name) {(symbol-name (*primary-keys-config* (keyword outer-key))) column-name})]
+ inner-object (or (inner-map column-name) {(symbol-name primary-key-name) (decode-with-key outer-key column-name)})]
(assoc hydrated outer-key
(assoc inner-map column-name
(assoc inner-object inner-key value)))))
@@ -182,5 +193,4 @@
(defn hbase-table [hbase-table-name]
(let [h-config (HBaseConfiguration.)
_ (.set h-config "hbase.master", *hbase-master*)]
- (HTable. h-config hbase-table-name)))
-
+ (HTable. h-config hbase-table-name)))

0 comments on commit ca8347b

Please sign in to comment.