@@ -91,5 +91,16 @@ (is (= (consumer "id") "103")) (is (= (merchant "name") "portable chairs")))) -(def sku-string "{\"merchant\": {\"id\": 11, \"name\": \"portable chairs\"}, \"active_campaigns\": [3, 4, 7, 10, 11, 13], \"consumer\": {\"kind\": \"visitor\", \"id\": 103, \"email_address\": \"f80a2173-5923-264f-e2d5-cb0f96220220@visitor.cinchcorp.com\"}, \"session\": {\"uber_session_id\": \"a96ec02e-0fd3-b030-c14a-1761ffe7d45b\", \"merchant_session_id\": \"3c0e276524dc843debaebfd9506138ee\", \"cinch_session_id\": \"42e00dc0eae2706a75eee1c1964d4d43\"}, \"api\": \"0.0.1.0\", \"inserts\": [{\"cinch_unit_price\": 36.95, \"campaign_id\": -1, \"html_id\": \"cinch_id_1231729409882\", \"merchant_product_id\": \"SS-REG\", \"sku\": \"SS-REG Black XL\", \"merchant_unit_price\": 36.95, \"insert_type\": \"campaign\"}]}") -(def sku-object (json/decode-from-str sku-string)) \ No newline at end of file +(def cart-string + "{\"event_type\": \"cart\", \"api\": \"0.0.1.0\", \"session\": {\"cinch_session_id\": \"1948c2cef48e58a6c1f215a545512077\", \"merchant_session_id\": \"b768e9911893b91b279e08ac8efde20f\", \"uber_session_id\": \"ee2bcc43-dae9-e0f0-40e8-325f96b5f114\"}, \"cart_items\": [{\"campaign_id\": -1, \"merchant_product_id\": \"EZ-30DC-HD\", \"merchant_unit_price\": 109.95, \"options\": null, \"name\": null, \"cinch_unit_price\": 109.95, \"sku\": \"EZ-30DC-HD (Yellow) XX\", \"quantity\": 1, \"description\": null}, {\"campaign_id\": -1, \"merchant_product_id\": \"OB-BW-LNGR\", \"merchant_unit_price\": 229.95, \"options\": null, \"name\": null, \"cinch_unit_price\": 229.95, \"sku\": \"OB-BW-LNGR (RED) LL\", \"quantity\": 1, \"description\": null}], \"page\": {\"merchant_template_name\": \"index\", \"referrer\": \"http://chairs.vasanta.hq.cinchcorp.com/shopping_cart.html?number_of_uploads=0\", \"request_url\": \"http://chairs.vasanta.hq.cinchcorp.com/hanging-chairs-c-23.html\"}, \"cart\": {\"checkout_state\": 1}, \"merchant\": {\"name\": \"Portable Folding Chairs\", \"id\": 14}, \"http_client\": {\"browser_version\": \"3\", \"operating_system\": \"Macintosh\", \"operating_system_version\": \"OS X\", \"ip_address\": \"192.168.10.10\", \"browser\": \"Safari\"}, \"active_campaigns\": [3, 4, 7, 10, 11, 13], \"consumer\": {\"email_address\": \"afd4cb68-bb3b-e51f-ac2a-cb6eefce528f@visitor.cinchcorp.com\", \"kind\": \"visitor\", \"id\": 103}}") +(def cart-object (json/decode-from-str cart-string)) + +(deftest test-flatten-with-sku + (let [flattened (flatten cart-object) + hydrated (hydrate flattened) + cart-items (hydrated "cart_items") + cart-item (first cart-items)] + (is (= (count cart-items) 2)) + (is (= (cart-item "sku") "OB-BW-LNGR (RED) LL")) + (is (= (cart-item "merchant_product_id") "OB-BW-LNGR")))) + \ No newline at end of file spec/org/rathore/amit/capjure_spec.clj @@ -1,27 +1,52 @@ (use 'clojure.contrib.test-is) -(import '(java.net URLEncoder)) +(import '(java.net URLEncoder URLDecoder)) (load-file "spec/org/rathore/amit/capjure_spec.clj") - (in-ns 'capjure-spec) (println "**************** running tests ******************") -;(def keys-config -; {:inserts :merchant_product_id -; :cart_items :merchant_product_id -; :latest_consumer :merchant_id}) +(defstruct key-config :qualifier :functor) +(defn add-key-config [keys-config name qualifier functor] + (assoc keys-config name (struct key-config qualifier functor))) +(defn encode-keys [] + (let [keys-config {} + keys-config (add-key-config keys-config + :inserts :merchant_product_id + (fn [insert-map] + (insert-map :merchant_product_id))) + keys-config (add-key-config keys-config + :latest_consumer :merchant_id + (fn [consumer-map] + (consumer-map :merchant_id))) + keys-config (add-key-config keys-config + :cart_items + :merchant_product_id (fn [cart-item-map] + (let [sku (cart-item-map :sku) + m-pid (cart-item-map :merchant_product_id)] + (cond + (nil? sku) m-pid + :else (str m-pid "@" (URLEncoder/encode sku "UTF-8"))))))] + keys-config)) + +(defn decode-keys [] + (let [keys-config {} + keys-config (add-key-config keys-config + :inserts :merchant_product_id + (fn [value] value)) + keys-config (add-key-config keys-config + :latest_consumer :merchant_id + (fn [value] value)) + keys-config (add-key-config keys-config + :cart_items + :merchant_product_id (fn [value] + (cond + (.contains value "@") (first (.split (URLDecoder/decode value "UTF-8") "@")) + :else value)))] + keys-config)) + +(def keys-config {:encode (encode-keys) :decode (decode-keys)}) (binding [*hbase-master* "tank.cinchcorp.com:60000" *primary-keys-config* keys-config] - (run-tests)) + (run-tests) (shutdown-agents)) - -(def keys-config { - :inserts (fn [insert-map] - (insert-map :merchant_product_id)) - :latest_consumer (fn [consumer-map] - (consumer-map :merchant_id)) - :cart_items (fn [cart-item-map] - (str (cart-item-map :merchant_product_id) "@" (URLEncoder/encode (cart-item-map :sku))))}) - -;create struct to track primary key, and function associated with it \ No newline at end of file spec_suite.clj @@ -4,11 +4,28 @@ '(org.apache.hadoop.hbase.client HTable Scanner) '(org.apache.hadoop.hbase.io BatchUpdate Cell)) - (def *mock-mode* false) (def *hbase-master* "localhost:60000") (def *primary-keys-config* {}) +(declare symbol-name) +(defn encoding-keys [] + (*primary-keys-config* :encode)) +(defn decoding-keys [] + (*primary-keys-config* :decode)) +(defn qualifier-for [key-name] + (((encoding-keys) key-name) :qualifier)) +(defn encoding-functor-for [key-name] + (((encoding-keys) key-name) :functor)) +(defn all-primary-keys [] + (map #(symbol-name %) (keys (encoding-keys)))) +(defn primary-key [column-family] + (first (filter #(.startsWith column-family (str %)) (all-primary-keys)))) +(defn decoding-functor-for [key-name] + (((decoding-keys) (keyword key-name)) :functor)) +(defn decode-with-key [key-name value] + ((decoding-functor-for key-name) value)) + (declare flatten add-to-insert-batch capjure-insert hbase-table read-row read-cell) (defn capjure-insert [object-to-save hbase-table-name row-id] (let [table (hbase-table hbase-table-name) @@ -63,10 +80,11 @@ :else (process-strings key (to-array all))))) (defn process-maps [key maps] - (let [qualifier (*primary-keys-config* key)] + (let [qualifier (qualifier-for key) + encoding-functor (encoding-functor-for key)] (apply merge (map (fn [single-map] - (process-map (symbol-name key) (single-map qualifier) (dissoc single-map qualifier))) + (process-map (symbol-name key) (encoding-functor single-map) (dissoc single-map qualifier))) maps)))) (defn process-map [initial-prefix final-prefix single-map] @@ -87,17 +105,10 @@ (seq bloated_object)))) (declare read-as-hash cell-value-as-string hydrate-pair has-many-strings-hydration has-many-objects-hydration has-one-string-hydration has-one-object-hydration collapse-for-hydration) - -(defn all-primary-keys [] - (map #(symbol-name %) (keys *primary-keys-config*))) - (defn is-from-primary-keys [key-name] (let [key-name-str (symbol-name key-name)] (some #(.startsWith key-name-str %) (all-primary-keys)))) -(defn primary-key [column-family] - (first (filter #(.startsWith column-family (str %)) (all-primary-keys)))) - (defn column-name-empty? [key-name] (= 1 (count (.split key-name ":")))) @@ -144,9 +155,9 @@ (defn has-many-objects-hydration [hydrated column-family column-name value] (let [outer-key (primary-key column-family) inner-key (.substring column-family (+ 1 (count outer-key)) (count column-family)) - primary-key-name (*primary-keys-config* outer-key) + primary-key-name (qualifier-for (keyword outer-key)) inner-map (or (hydrated outer-key) {}) - inner-object (or (inner-map column-name) {(symbol-name (*primary-keys-config* (keyword outer-key))) column-name})] + inner-object (or (inner-map column-name) {(symbol-name primary-key-name) (decode-with-key outer-key column-name)})] (assoc hydrated outer-key (assoc inner-map column-name (assoc inner-object inner-key value))))) @@ -182,5 +193,4 @@ (defn hbase-table [hbase-table-name] (let [h-config (HBaseConfiguration.) _ (.set h-config "hbase.master", *hbase-master*)] - (HTable. h-config hbase-table-name))) - \ No newline at end of file + (HTable. h-config hbase-table-name))) \ No newline at end of file src/org/rathore/amit/capjure.clj 2bb2215686f1a0f1ff0a7ff1e9484e3d99d90933 amitrathore amitrathore@gmail.com http://github.com/amitrathore/capjure/commit/ca8347b9006f05b2a55d32ae6ecfdbfc7195f87f ca8347b9006f05b2a55d32ae6ecfdbfc7195f87f 2009-02-02T17:37:46-08:00 2009-02-02T17:37:46-08:00 added the possibility of specifying encoding/decoding functions when flattening objects to be stored in hbase a4ea4516d0b49fdd0a5bc4b209a48bc39210066b amitrathore amitrathore@gmail.com