Permalink
Browse files

Update couchdb test with bulk loading fixed from Paul and Chris

  • Loading branch information...
1 parent 87ab683 commit 03f09a2d6d250ed7069834fb24a4438b2fac4687 @chapmanb chapmanb committed May 11, 2009
Showing with 19 additions and 15 deletions.
  1. +6 −10 keyval_testing/couchdb_get_freqs.py
  2. +13 −5 keyval_testing/freq_to_couchdb.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python
"""Get frequencies from a CouchDB remote database.
-Total time : 14:08.47s
-Memory : 8112
-Percent CPU : 63.2%
+Total time : 14:22.19s
+Memory : 8154
+Percent CPU : 63.8%
"""
import sys
import random
@@ -14,18 +14,14 @@ def main():
server = couchdb.client.Server("http://mothra:5984/")
db = server[db_name]
- max_records = 2810718
+ max_records = 2810717
num_trials = 500000
- misses = 0
for index in range(num_trials):
read_id = str(random.randint(0, max_records))
- try:
- doc = db[read_id]
- except:
- misses += 1
+ doc = db[read_id]
freq = int(doc["frequency"])
if index % 10000 == 0:
- print index, misses, read_id, freq
+ print index, read_id, freq
if __name__ == "__main__":
main()
@@ -1,7 +1,7 @@
#!/usr/bin/env python
"""Write a file of read frequencies to a CouchDB database.
-Did not finish:
+Did not finish on first pass with naive code:
Total time : 22:15:19.10s
Memory : 4896
@@ -10,6 +10,14 @@
-rw-rw-r-- 1 chapman users 5.7G 2009-05-07 07:52 read_to_freq.couch
1842694 documents loaded
+
+Re-done with improvements from Chris and Paul for bulk loading:
+
+ Total time : 5:31.01s
+ Memory : 7490
+ Percent CPU : 10.4%
+
+ -rw-rw-r-- 1 chapman users 236M 2009-05-11 08:01 read_to_freq.couch
"""
import sys
import couchdb.client
@@ -27,10 +35,10 @@ def main(in_file):
with open(in_file) as in_handle:
for read_index, freq in enumerate(in_handle):
- bulk_docs.append(dict(_id=read_index, frequency=freq))
- if len(bulk_docs) >= bulk_size
- db.update(bulk_docs)
- bulk_docs = []
+ bulk_docs.append(dict(_id=str(read_index), frequency=int(freq)))
+ if len(bulk_docs) >= bulk_size:
+ db.update(bulk_docs)
+ bulk_docs = []
db.update(bulk_docs)
if __name__ == "__main__":

0 comments on commit 03f09a2

Please sign in to comment.