Permalink
Browse files

Save.

  • Loading branch information...
drewr committed Sep 8, 2011
1 parent ca092d0 commit 1f8dc3db0b8ff7104300fb9c7f53bd87cabc9e13
Showing with 45 additions and 19 deletions.
  1. +45 −19 src/esperanto/stress.clj
View
@@ -1,6 +1,7 @@
(ns esperanto.stress
(:require [clojure.java.io :as io]
[clojure.string :as s]
+ [cheshire.core :as json]
[esperanto.search :as search])
(:use [esperanto.index :only [index-bulk]]
[esperanto.admin.indices :only [refresh]]
@@ -10,39 +11,64 @@
(repeatedly n (fn [] (rand-nth xs))))
(def words
- (take 10000
+ (take 50000
(-> "/usr/share/dict/words" io/reader line-seq)))
-(defn take-docs [corpus x y]
- (repeatedly x (fn []
+(defn take-docs [n m xs]
+ (repeatedly n (fn []
{"type" "random"
- "text" (s/join " " (take-rand y corpus))})))
+ "text" (s/join " " (take-rand m xs))})))
(defn index-rand [client idx corpus x y]
- (index-bulk client idx (take-docs corpus x y)))
+ (index-bulk client idx (take-docs x y corpus)))
+
+(defn save-doc [f doc]
+ (locking save-doc
+ (with-open [f (io/writer f :append true)]
+ (.append f (json/generate-string doc))
+ (.newLine f))))
+
+
(comment
+ (def docs "docs.txt")
+
+ ;; Store docs
+ (time (count (pmap #(save-doc docs %) (take-docs 1000 500 words))))
+
(def foo (-> (make-client-node
{"node.name" "Foo!!!"
"discovery.zen.ping.multicast.enabled" "false"
"discovery.zen.ping.unicast.hosts" "localhost:9300"
"cluster.name" "elasticsearch"})
.start))
- (time
- (let [c (.client foo)
- me (System/currentTimeMillis)
- iter 10
- bsize 500
- wsize 500
- t (atom 0)]
- (println me "start" iter bsize wsize)
- (dotimes [_ iter]
- (index-rand c "test" words bsize wsize)
- (refresh c "test")
- (println me (swap! t + batch)))
- (println me "end" (* iter bsize) @t)))
+ (let [c (.client foo)
+ now (System/currentTimeMillis)
+ iter 10
+ bsize 500
+ wsize 500
+ t (atom 0)]
+ (println now "start" iter bsize wsize)
+ (dotimes [_ iter]
+ (index-rand c "test" words bsize wsize)
+ (refresh c "test")
+ (println now (swap! t + batch) (- (System/currentTimeMillis) now)))
+ (println now "end" (* iter bsize) @t (- (System/currentTimeMillis) now)))
+
+ (let [c (.client foo)
+ idx "test"
+ now (System/currentTimeMillis)
+ bsize 100]
+ (println now "start")
+ (doseq [docs (partition-all bsize
+ (map json/parse-string
+ (line-seq (io/reader docs))))]
+ (index-bulk c idx docs)
+ (refresh c idx)
+ (println now (- (System/currentTimeMillis) now) (search/count c idx)))
+ (println now "end" (- (System/currentTimeMillis) now) (search/count c idx)))
- )
+)

0 comments on commit 1f8dc3d

Please sign in to comment.