Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 187 lines (150 sloc) 6.286 kb
d038539 @steveyen Initial couch-spill test command.
steveyen authored
1 #!/usr/bin/env python
2
64c5085 @steveyen Couch spill reader corrected
steveyen authored
3 import re
d038539 @steveyen Initial couch-spill test command.
steveyen authored
4 import sys
5 import copy
6 import math
7 import time
8 import socket
9 import threading
10 import mcsoda
11
12 class Reader(threading.Thread):
64c5085 @steveyen Couch spill reader corrected
steveyen authored
13 def __init__(self, src, reader_go, reader_done):
d038539 @steveyen Initial couch-spill test command.
steveyen authored
14 self.src = src
64c5085 @steveyen Couch spill reader corrected
steveyen authored
15 self.reader_go = reader_go
16 self.reader_done = reader_done
17 self.inflight = 0
a6910fe @steveyen Track network bytes sent/recv and report transfer rates
steveyen authored
18 self.received = 0
d038539 @steveyen Initial couch-spill test command.
steveyen authored
19 threading.Thread.__init__(self)
20
21 def run(self):
64c5085 @steveyen Couch spill reader corrected
steveyen authored
22 self.reader_go.wait()
23 self.reader_go.clear()
24 while True:
25 data = self.src.recv(4096)
26 if not data:
27 break
28
a6910fe @steveyen Track network bytes sent/recv and report transfer rates
steveyen authored
29 self.received += len(data)
30
64c5085 @steveyen Couch spill reader corrected
steveyen authored
31 found = len(re.findall("HTTP/1.1 ", data))
32
33 self.inflight -= found
34 if self.inflight == 0:
35 self.reader_done.set()
36 self.reader_go.wait()
37 self.reader_go.clear()
d038539 @steveyen Initial couch-spill test command.
steveyen authored
38
39
40 # Stream some mcsoda onto a couch for performance testing.
41 #
42 class StoreCouch(mcsoda.Store):
43
44 def connect(self, target, user, pswd, cfg, cur):
45 self.cfg = cfg
46 self.cur = cur
47 self.target = target
48 self.host_port = (target + ":5984").split(':')[0:2]
49 self.host_port[1] = int(self.host_port[1])
50 self.queue = []
51 self.ops = 0
fe69cdd @steveyen Use seqno of 1 when only creating docs in couch-spill.
steveyen authored
52 self.seq = 0
d038539 @steveyen Initial couch-spill test command.
steveyen authored
53 self.skt = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
54 self.skt.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
55 self.skt.connect(tuple(self.host_port))
64c5085 @steveyen Couch spill reader corrected
steveyen authored
56 self.reader_go = threading.Event()
57 self.reader_done = threading.Event()
a6910fe @steveyen Track network bytes sent/recv and report transfer rates
steveyen authored
58 self.reader = Reader(self.skt, self.reader_go, self.reader_done)
59 self.reader.daemon = True
60 self.reader.start()
61 self.xfer_sent = 0
62 self.xfer_recv = 0
d038539 @steveyen Initial couch-spill test command.
steveyen authored
63
e13293a @steveyen Specialized showing some keys
steveyen authored
64 def show_some_keys(self):
65 pass
66
d038539 @steveyen Initial couch-spill test command.
steveyen authored
67 def gen_doc(self, key_num, key_str, min_value_size, json=True, cache=None):
68 # Always json and never cache.
69 #
fc44146 @steveyen Generate rev-id's that ep-engine will like
steveyen authored
70 # seqno : 4 bytes
71 # cas : 8 bytes
72 # length : 4 bytes
73 # flags : 4 bytes
74 #
d038539 @steveyen Initial couch-spill test command.
steveyen authored
75 self.seq = self.seq + 1
fe69cdd @steveyen Use seqno of 1 when only creating docs in couch-spill.
steveyen authored
76 seqno = self.seq
77 if self.cfg.get('ratio-sets', 0) >= 1.0 and \
78 self.cfg.get('ratio-creates', 0) >= 1.0:
79 # If we're only creating docs, then seqno should be 1.
80 seqno = 1
81
82 # suffix_ex = '"_rev":"1-0286dbb6323b61e7f0be3ba5d1633985",'
83 suffix_ex = '"_rev":"%s-00000000000000000000000000000000",' % (seqno,)
d038539 @steveyen Initial couch-spill test command.
steveyen authored
84
fc44146 @steveyen Generate rev-id's that ep-engine will like
steveyen authored
85 d = mcsoda.gen_doc_string(key_num, key_str, min_value_size,
86 self.cfg['suffix'][min_value_size],
87 True, cache=None, key_name="_id",
88 suffix_ex=suffix_ex,
89 whitespace=False)
6f3af7d @steveyen Don't count meta-key length in rev-id length
steveyen authored
90
91 strip = len('"_id":"00003e3b9e533668",') + len(suffix_ex)
92 dlen = len(d) - strip
93 dlen = hex(dlen)[2:].rjust(8, '0')
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
94 suff = '-0000000000000000%s00000000' % (dlen,)
95 d = d.replace("-00000000000000000000000000000000", suff)
96
fe69cdd @steveyen Use seqno of 1 when only creating docs in couch-spill.
steveyen authored
97 return (d, str(seqno) + suff) # Returns (doc, rev-id).
d038539 @steveyen Initial couch-spill test command.
steveyen authored
98
99 def command(self, c):
100 self.queue.append(c)
101 if len(self.queue) > (self.cur.get('batch') or \
102 self.cfg.get('batch', 100)):
103 self.flush()
104 return True
105 return False
106
107 def flush(self):
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
108 revs_diff_arr = [ "POST /default/_revs_diff HTTP/1.1\r\n" \
109 "Content-Type: application/json\r\n" \
110 "Accept: application/json\r\n" \
111 "Host: %s:%s\r\n" % (self.host_port[0], self.host_port[1]),
112 "Content-Length: ", None, "\r\n\r\n",
113 '{' ]
114 revs_diff_len = len(revs_diff_arr[-1]) # Content length.
115
11df6fc @steveyen Better variable names
steveyen authored
116 bulk_docs_arr = [ "POST /default/_bulk_docs HTTP/1.1\r\n" \
117 "X-Couch-Full-Commit: false\r\n" \
118 "Content-Type: application/json\r\n" \
119 "Accept: application/json\r\n" \
120 "Host: %s:%s\r\n" % (self.host_port[0], self.host_port[1]),
121 "Content-Length: ", None, "\r\n\r\n",
122 '{"new_edits":false,"docs":[' ]
123 bulk_docs_len = len(bulk_docs_arr[-1]) # Content length.
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
124
125 docs_num = 0 # Number of actual docs to be sent.
126
d038539 @steveyen Initial couch-spill test command.
steveyen authored
127 for c in self.queue:
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
128 cmd, key_num, key_str, doc_rev, expiration = c
129 if doc_rev:
130 doc, rev = doc_rev
131
132 if docs_num > 0:
133 revs_diff_arr.append(',')
134 revs_diff_len += 1
135
11df6fc @steveyen Better variable names
steveyen authored
136 bulk_docs_arr.append(',')
137 bulk_docs_len += 1
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
138
139 x = '"%s":["%s"]' % (key_str, rev)
140 revs_diff_arr.append(x)
141 revs_diff_len += len(x)
142
11df6fc @steveyen Better variable names
steveyen authored
143 bulk_docs_arr.append(doc)
144 bulk_docs_len += len(doc)
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
145
146 docs_num += 1
147
148 revs_diff_arr.append("}")
149 revs_diff_len += 1
150
11df6fc @steveyen Better variable names
steveyen authored
151 bulk_docs_arr.append("]}")
152 bulk_docs_len += 2
153
939595d @steveyen Generate revs-diffs protocol commands
steveyen authored
154 if docs_num > 0:
155 revs_diff_arr[2] = str(revs_diff_len) # Fill content length placeholders.
156 bulk_docs_arr[2] = str(bulk_docs_len)
157
6c811eb @steveyen Use separate sends()'s for revs_difss and bulk_docs in couch-spill.
steveyen authored
158 if True:
159 # Use separate sends()'s for revs_diff and bulk_docs.
160 to_send = [(''.join(revs_diff_arr), 1), (''.join(bulk_docs_arr), 1)]
161 else:
162 # Send revs_diffs + bulk_docs in one send().
163 to_send = [(''.join(revs_diff_arr) + ''.join(bulk_docs_arr), 2)]
164
165 for m, num_posts in to_send:
166 self.reader.inflight += num_posts
167 self.skt.send(m)
168 self.xfer_sent += len(m)
169
170 r = self.reader.received
171 self.reader_go.set()
172 self.reader_done.wait()
173 self.reader_done.clear()
174 self.xfer_recv += self.reader.received - r
d038539 @steveyen Initial couch-spill test command.
steveyen authored
175
176 self.ops += len(self.queue)
177 self.queue = []
178
179
180 if __name__ == "__main__":
181 if sys.argv[1].find("http") != 0:
182 raise Exception("usage: %s http://HOST:5984 ..." % (sys.argv[0],))
183
184 argv = (' '.join(sys.argv) + ' doc-gen=0').split(' ')
185
186 mcsoda.main(argv, protocol="http", stores=[StoreCouch()])
Something went wrong with that request. Please try again.