Permalink
Browse files

Support sending requests trough multiples slots in QPS exmaple spider

  • Loading branch information...
1 parent fd58de4 commit e676ad31d049a5718c26d12dde3076366b85b786 @dangra dangra committed Dec 5, 2012
Showing with 21 additions and 8 deletions.
  1. +3 −3 extras/qps-bench-server.py
  2. +18 −5 extras/qpsclient.py
@@ -39,9 +39,9 @@ def render(self, request):
qps = len(self.tail) / sum(self.tail)
print('samplesize={0} concurrent={1} qps={2:0.2f}'.format(len(self.tail), self.concurrent, qps))
- if 'sleep' in request.args:
- sleep = float(request.args['sleep'][0])
- reactor.callLater(sleep, self._finish, request)
+ if 'latency' in request.args:
+ latency = float(request.args['latency'][0])
+ reactor.callLater(latency, self._finish, request)
return NOT_DONE_YET
self.concurrent -= 1
View
@@ -3,7 +3,7 @@
usage:
- scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a sleep=0.3
+ scrapy runspider qpsclient.py --loglevel=INFO --set RANDOMIZE_DOWNLOAD_DELAY=0 --set CONCURRENT_REQUESTS=50 -a qps=10 -a latency=0.3
"""
@@ -16,27 +16,40 @@ class QPSSpider(BaseSpider):
name = 'qps'
benchurl = 'http://localhost:8880/'
-
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
+ download_delay = None
# time in seconds to delay server responses
- sleep = None
+ latency = None
+ # number of slots to create
+ slots = 1
def __init__(self, *a, **kw):
super(QPSSpider, self).__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
+ elif self.download_delay is not None:
+ self.download_delay = float(self.download_delay)
def start_requests(self):
url = self.benchurl
- if self.sleep is not None:
- url += '?sleep={0}'.format(self.sleep)
+ if self.latency is not None:
+ url += '?latency={0}'.format(self.latency)
+
+ slots = int(self.slots)
+ if slots > 1:
+ urls = [url.replace('localhost', '127.0.0.%d' % (x + 1)) for x in xrange(slots)]
+ else:
+ urls = [url]
+ idx = 0
while True:
+ url = urls[idx % len(urls)]
yield Request(url, dont_filter=True)
+ idx += 1
def parse(self, response):
pass

0 comments on commit e676ad3

Please sign in to comment.