Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 292 lines (252 sloc) 10.689 kb
fff48063 »
2011-07-15 Initial commit.
1 # Author: Ovidiu Predescu
2 # Date: July 2011
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License"); you may
5 # not use this file except in compliance with the License. You may obtain
6 # a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15 """
16 A simple benchmark for various asynchronous Python MySQL client libraries.
17 """
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
18
19 import sys
fff48063 »
2011-07-15 Initial commit.
20 import time
21
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
22 from adisp import process
23 import adb
24
fff48063 »
2011-07-15 Initial commit.
25 import tornado.options
26 from tornado.options import define, options
27
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
28 define("dbhost", default="127.0.0.1", help="Database host")
29 define("dbuser", default="", help="Database user to use")
30 define("dbpasswd", default="", help="User's password")
31 define("db", default="test", help="Database to use")
fff48063 »
2011-07-15 Initial commit.
32
33 define("use_tornado", default=False,
34 help="Use tornado twisted reactor instead of twisted's reactor")
35
36 define("use_txmysql", default=False, help="Use txMySQL database module")
37 define("use_adbapi", default=False, help="Use twisted's adbapi module")
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
38 define("use_adb", default=False, help="Use our own adb module")
c7feafe0 »
2011-08-30 Added a quick hack to test against momoko.
39 define("use_momoko", default=False, help="Use the momoko module")
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
40
41 define("use_postgres", default=False, help="Use Postgres with psycopg2")
42 define("use_mysql", default=False, help="Use MySQL")
fff48063 »
2011-07-15 Initial commit.
43
44 define("pool_size", default=10, help="Database connection pool size")
45
46 class DbBenchmark:
47 def __init__(self):
48 self._pool = None
49
50 def run(self):
51 d = self._createTable()
52 d.addCallback(self._startBenchmark)
53 d.addErrback(self._dbError)
54
55 def _dbError(self, error):
56 print "Error accessing the database: %s" % error
57
58 def _createTable(self):
59 print 'creating benchmark table'
60 return self._pool.runOperation(
61 """drop table if exists benchmark;
62 create table benchmark(
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
63 id INT NOT NULL PRIMARY KEY,
fff48063 »
2011-07-15 Initial commit.
64 data VARCHAR(100)
65 )
66 """)
67
68 def _startBenchmark(self, ignored):
69 print "Starting benchmark %s" % self.__class__
70 self._start_time = time.time()
71 self._totalNumInserts = 100000
72 self._numDone = 0
73 for i in xrange(self._totalNumInserts):
74 d = self._doInsert(i)
75 d.addCallback(self._insertDone)
76 d.addErrback(self._dbError)
77
78 def _doInsert(self, i):
79 raise NotImplementedError
80
81 def _insertDone(self, ignored):
82 self._numDone += 1
83 if self._numDone == self._totalNumInserts:
84 d = self._pool.runQuery("select count(*) from benchmark")
85 d.addCallback(self._allInsertsDone)
86 d.addErrback(self._dbError)
87
88 def _allInsertsDone(self, results):
89 self._end_time = time.time()
90 row = results[0]
91 num = row[0]
92 print "inserted %d records, time taken = %f seconds" %\
93 (num, (self._end_time - self._start_time))
94 reactor.stop()
95
96 class TwistedDbAPI(DbBenchmark):
97 def __init__(self):
98 from twisted.enterprise import adbapi
99 print "Creating twisted adbapi ConnectionPool"
100 self._pool = adbapi.ConnectionPool(dbapiName="MySQLdb",
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
101 host=options.dbhost,
fff48063 »
2011-07-15 Initial commit.
102 port=3306,
103 unix_socket='',
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
104 user=options.dbuser,
105 passwd=options.dbpasswd,
106 db=options.db,
fff48063 »
2011-07-15 Initial commit.
107 cp_min=options.pool_size,
108 cp_max=options.pool_size)
109 self._numRuns = 0
110
111 def _doInsert(self, i):
112 return self._pool.runOperation(
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
113 "insert benchmark (id, data) values (%s, %s)" % (i, i))
fff48063 »
2011-07-15 Initial commit.
114
115
116 class TxMySQL(DbBenchmark):
117 def __init__(self):
118 from txmysql import client
119 print "Creating txMySQL ConnectionPool"
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
120 self._pool = client.ConnectionPool(hostname=options.dbhost,
121 username=options.dbuser,
122 password=options.dbpasswd,
123 database=options.db,
fff48063 »
2011-07-15 Initial commit.
124 num_connections=options.pool_size)
125
126 def _doInsert(self, i):
127 return self._pool.runOperation(
128 "insert benchmark(data) values (%d)" % i)
129
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
130 class TwistedDbPostgresAPI(DbBenchmark):
131 def __init__(self):
132 from twisted.enterprise import adbapi
133 print "Creating twisted adbapi ConnectionPool"
134 self._pool = adbapi.ConnectionPool(dbapiName="psycopg2",
135 host=options.dbhost,
136 user=options.dbuser,
137 password=options.dbpasswd,
138 database=options.db,
139 cp_min=options.pool_size,
140 cp_max=options.pool_size)
141 self._numRuns = 0
142
143 def _doInsert(self, i):
144 return self._pool.runOperation(
145 "insert into benchmark (id, data) values (%s, %s)" % (i, i))
146
147
148 class AsyncDatabaseBenchmark:
149 def __init__(self, driver,
150 database=None, user=None, password=None, host=None):
151 self.adb = adb.Database(driver=driver, database=database, user=user,
152 password=password, host=host,
153 num_threads=options.pool_size,
154 queue_timeout=0.1)
155
156 def run(self):
157 self.ioloop = tornado.ioloop.IOLoop.instance()
158 self.ioloop.add_callback(self.whenRunningCallback)
159
160 @process
161 def whenRunningCallback(self):
162 # Drop the table if it exists
163 yield self.adb.runOperation("drop table if exists benchmark")
164 yield self.adb.runOperation("""
165 create table benchmark (
166 userid int not null primary key,
167 data VARCHAR(100)
168 );
169 """)
170 rows_to_insert = 100000
171 # Insert some rows
172 start_time = time.time()
173 stmts = []
174 for i in xrange(rows_to_insert):
175 stmts.append(
176 ("insert into benchmark (userid, data) values (%s, %s)",
177 (i, i)))
178 numrows = yield map(self.adb.runOperation, stmts)
179 end_time = time.time()
180
181 rows = yield self.adb.runQuery("select count(*) from benchmark")
182 print 'inserted %s records, time taken = %s seconds' % \
183 (rows, end_time - start_time)
184 self.ioloop.stop()
185 self.adb.stop()
fff48063 »
2011-07-15 Initial commit.
186
c7feafe0 »
2011-08-30 Added a quick hack to test against momoko.
187 try:
188 # Quick hack to test agains momoko: http://momoko.61924.nl/
189 import momoko
190 has_momoko = True
191
192 class MomokoDatabaseBenchmark:
193 def __init__(self, database=None, user=None, password=None, host=None):
194 self.adb = momoko.AdispClient({
195 'host': 'localhost',
196 'database': database,
197 'user': user,
198 'password': password,
199 'min_conn': 10,
200 'max_conn': 10,
201 'cleanup_timeout': 10
202 })
203
204 def run(self):
205 self.ioloop = tornado.ioloop.IOLoop.instance()
206 self.ioloop.add_callback(self.whenRunningCallback)
207
208 @momoko.process
209 def whenRunningCallback(self):
210 # Drop the table if it exists
211 yield self.adb.execute("drop table if exists benchmark")
212 yield self.adb.execute("""
213 create table benchmark (
214 userid int not null primary key,
215 data VARCHAR(100)
216 );
217 """)
218 rows_to_insert = 100000
219 # Insert some rows
220 start_time = time.time()
221 stmts = []
222 for i in xrange(rows_to_insert):
223 yield self.adb.execute(
224 "insert into benchmark (userid, data) values (%s, %s)",
225 (i, i))
226 end_time = time.time()
227
228 cursor = yield self.adb.execute("select count(*) from benchmark")
229 rows = cursor.fetchall()
230 print 'inserted %s records, time taken = %s seconds' % \
231 (rows, end_time - start_time)
232 self.ioloop.stop()
233 self.adb.close()
234
235 except ImportError:
236 has_momoko = False
237
fff48063 »
2011-07-15 Initial commit.
238 if __name__ == "__main__":
239 tornado.options.parse_command_line()
240 if options.use_tornado:
d6fcd8e0 »
2011-07-20 Based the code on the latest Tornado that incorporates the twisted-st…
241 import tornado.platform.twisted
242 tornado.platform.twisted.install()
fff48063 »
2011-07-15 Initial commit.
243 from twisted.internet import reactor
244
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
245 benchmark = None
fff48063 »
2011-07-15 Initial commit.
246 if options.use_adbapi:
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
247 if options.use_postgres:
248 benchmark = TwistedDbPostgresAPI()
249 elif options.use_mysql:
250 benchmark = TwistedDbAPI()
fff48063 »
2011-07-15 Initial commit.
251 elif options.use_txmysql:
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
252 # This only works with MySQL
fff48063 »
2011-07-15 Initial commit.
253 benchmark = TxMySQL()
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
254 elif options.use_adb:
255 if options.use_postgres:
256 driver = "psycopg2"
257 elif options.use_mysql:
258 driver = "MySQLdb"
259 else:
260 print 'Use --use_postgres or --use_mysql to specify database.'
261 sys.exit(1)
262 benchmark = AsyncDatabaseBenchmark(driver,
263 database=options.db,
264 user=options.dbuser,
265 password=options.dbpasswd,
266 host=options.dbhost)
c7feafe0 »
2011-08-30 Added a quick hack to test against momoko.
267 elif options.use_momoko:
268 if not has_momoko:
269 print 'Could not find momoko in PYTHONPATH'
270 sys.exit(1)
271 benchmark = MomokoDatabaseBenchmark(database=options.db,
272 user=options.dbuser,
273 password=options.dbpasswd,
274 host=options.dbhost)
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
275
276 if benchmark:
fff48063 »
2011-07-15 Initial commit.
277 benchmark.run()
278 else:
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
279 print 'Could not find a useful combination of options'
fff48063 »
2011-07-15 Initial commit.
280 print 'At least one of --use_adbapi or --use_txmysql should be '\
281 'specified'
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
282 sys.exit(1)
283
c7feafe0 »
2011-08-30 Added a quick hack to test against momoko.
284 if options.use_adb or options.use_momoko:
967d5b39 »
2011-08-26 Added my own adb.py asynchronous database API running on top of Tornado.
285 print "Using Tornado IOLoop directly"
286 tornado.ioloop.IOLoop.instance().start()
287 else:
288 reactor.suggestThreadPoolSize(options.pool_size)
289 print "Using reactor %s" % reactor
290 reactor.run()
291
Something went wrong with that request. Please try again.