Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

tries to address issue #29 #52

Merged
merged 14 commits into from

2 participants

@deanmalmgren

The existing statistical reporting is a little misleading, particularly the "50%" and "90%" values. The title "50% response times: 123.45 ms" suggests that 50% of all of the responses across all of the bees is 123.45 ms. This is not what is calculated, but that is what would be useful to know.

This series of change sets attempts to address this issue by using ab -e, which reports a csv file of the full percentile results. We then aggregate this in the correct way so we can finally specify the overall median ("50%") and 90th percentile values, which are really useful to know.

This change set also includes the -e/--csv option for attack, to allow the user to see the full percentile statistics across all bees and each of the reporting bees. This is useful for detecting if one of the bees, for example, has a crappy connection.

Sorry there are so many change sets in here. I tried to break each one into manageable chunks that logically corresponded with the changes that were made. If there's anything else I can do to clarify this, please let me know.

@deanmalmgren deanmalmgren doh. much easier to do this by sampling rather than the confusing way…
… of converting to pdf first. added benefity of handling small data sets in a much better way. +1
d02aae2
@cosmin
Collaborator

Looks great, but it doesn't currently merge cleanly. If you can resolve the conflicts I'll get this merged in tomorrow.

@deanmalmgren
@cosmin
Collaborator

Try to rebase to master, hopefully any issues you saw earlier have been resolved. Otherwise please open an issue and I'll get on it ASAP.

@deanmalmgren

OK, that should do it. There were a few broken things that I fixed with the last two commits, but I've been able to confirm that everything works with the -e/--csv flag. If there's anything else, let me know. Cheers!

@cosmin cosmin merged commit 4bf7b2d into from
@deanmalmgren deanmalmgren deleted the branch
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Apr 23, 2013
  1. @deanmalmgren
  2. @deanmalmgren
  3. @deanmalmgren
  4. @deanmalmgren
  5. @deanmalmgren
  6. @deanmalmgren
  7. @deanmalmgren
  8. @deanmalmgren
  9. @deanmalmgren
  10. @deanmalmgren
  11. @deanmalmgren

    doh. much easier to do this by sampling rather than the confusing way…

    deanmalmgren authored
    … of converting to pdf first. added benefity of handling small data sets in a much better way. +1
Commits on Apr 27, 2013
  1. @deanmalmgren
  2. @deanmalmgren
  3. @deanmalmgren
This page is out of date. Refresh to see the latest.
View
1  .gitignore
@@ -5,3 +5,4 @@ theswarm.txt
build
MANIFEST
dist
+env
View
95 beeswithmachineguns/bees.py
@@ -30,6 +30,9 @@
import socket
import time
import urllib2
+import csv
+import math
+import random
import boto
import boto.ec2
@@ -210,11 +213,20 @@ def _attack(params):
for h in params['headers'].split(';'):
options += ' -H "%s"' % h
+ stdin, stdout, stderr = client.exec_command('tempfile -s .csv')
+ params['csv_filename'] = stdout.read().strip()
+ if params['csv_filename']:
+ options += ' -e %(csv_filename)s' % params
+ else:
+ print 'Bee %i lost sight of the target (connection timed out creating csv_filename).' % params['i']
+ return None
+
if params['post_file']:
os.system("scp -q -o 'StrictHostKeyChecking=no' %s %s@%s:/tmp/honeycomb" % (params['post_file'], params['username'], params['instance_name']))
options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
- benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options) "%(url)s"' % params
+ params['options'] = options
+ benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
stdin, stdout, stderr = client.exec_command(benchmark_command)
response = {}
@@ -223,22 +235,27 @@ def _attack(params):
ms_per_request_search = re.search('Time\ per\ request:\s+([0-9.]+)\ \[ms\]\ \(mean\)', ab_results)
if not ms_per_request_search:
- print 'Bee %i lost sight of the target (connection timed out).' % params['i']
+ print 'Bee %i lost sight of the target (connection timed out running ab).' % params['i']
return None
requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results)
failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results)
- fifty_percent_search = re.search('\s+50\%\s+([0-9]+)', ab_results)
- ninety_percent_search = re.search('\s+90\%\s+([0-9]+)', ab_results)
complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results)
response['ms_per_request'] = float(ms_per_request_search.group(1))
response['requests_per_second'] = float(requests_per_second_search.group(1))
response['failed_requests'] = float(failed_requests.group(1))
- response['fifty_percent'] = float(fifty_percent_search.group(1))
- response['ninety_percent'] = float(ninety_percent_search.group(1))
response['complete_requests'] = float(complete_requests_search.group(1))
+ stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params)
+ response['request_time_cdf'] = []
+ for row in csv.DictReader(stdout):
+ row["Time in ms"] = float(row["Time in ms"])
+ response['request_time_cdf'].append(row)
+ if not response['request_time_cdf']:
+ print 'Bee %i lost sight of the target (connection timed out reading csv).' % params['i']
+ return None
+
print 'Bee %i is out of ammo.' % params['i']
client.close()
@@ -247,8 +264,7 @@ def _attack(params):
except socket.error, e:
return e
-
-def _print_results(results):
+def _print_results(results, params, csv_filename):
"""
Print summarized load-testing results.
"""
@@ -256,6 +272,10 @@ def _print_results(results):
exception_bees = [r for r in results if type(r) == socket.error]
complete_bees = [r for r in results if r is not None and type(r) != socket.error]
+ timeout_bees_params = [p for r,p in zip(results, params) if r is None]
+ exception_bees_params = [p for r,p in zip(results, params) if type(r) == socket.error]
+ complete_bees_params = [p for r,p in zip(results, params) if r is not None and type(r) != socket.error]
+
num_timeout_bees = len(timeout_bees)
num_exception_bees = len(exception_bees)
num_complete_bees = len(complete_bees)
@@ -280,19 +300,31 @@ def _print_results(results):
complete_results = [r['requests_per_second'] for r in complete_bees]
mean_requests = sum(complete_results)
- print ' Requests per second:\t%f [#/sec] (mean)' % mean_requests
+ print ' Requests per second:\t%f [#/sec]' % mean_requests
complete_results = [r['ms_per_request'] for r in complete_bees]
mean_response = sum(complete_results) / num_complete_bees
- print ' Time per request:\t\t%f [ms] (mean)' % mean_response
-
- complete_results = [r['fifty_percent'] for r in complete_bees]
- mean_fifty = sum(complete_results) / num_complete_bees
- print ' 50%% response time:\t\t%f [ms] (mean)' % mean_fifty
-
- complete_results = [r['ninety_percent'] for r in complete_bees]
- mean_ninety = sum(complete_results) / num_complete_bees
- print ' 90%% response time:\t\t%f [ms] (mean)' % mean_ninety
+ print ' Time per request:\t\t%f [ms] (mean of bees)' % mean_response
+
+ # Recalculate the global cdf based on the csv files collected from
+ # ab. Can do this by sampling the request_time_cdfs for each of
+ # the completed bees in proportion to the number of
+ # complete_requests they have
+ n_final_sample = 100
+ sample_size = 100*n_final_sample
+ n_per_bee = [int(r['complete_requests']/total_complete_requests*sample_size)
+ for r in complete_bees]
+ sample_response_times = []
+ for n, r in zip(n_per_bee, complete_bees):
+ cdf = r['request_time_cdf']
+ for i in range(n):
+ j = int(random.random()*len(cdf))
+ sample_response_times.append(cdf[j]["Time in ms"])
+ sample_response_times.sort()
+ request_time_cdf = sample_response_times[0:sample_size:sample_size/n_final_sample]
+
+ print ' 50%% responses faster than:\t%f [ms]' % request_time_cdf[49]
+ print ' 90%% responses faster than:\t%f [ms]' % request_time_cdf[89]
if mean_response < 500:
print 'Mission Assessment: Target crushed bee offensive.'
@@ -305,12 +337,33 @@ def _print_results(results):
else:
print 'Mission Assessment: Swarm annihilated target.'
+ if csv_filename:
+ with open(csv_filename, 'w') as stream:
+ writer = csv.writer(stream)
+ header = ["% faster than", "all bees [ms]"]
+ for p in complete_bees_params:
+ header.append("bee %(instance_id)s [ms]" % p)
+ writer.writerow(header)
+ for i in range(100):
+ row = [i, request_time_cdf[i]]
+ for r in results:
+ row.append(r['request_time_cdf'][i]["Time in ms"])
+ writer.writerow(row)
+
def attack(url, n, c, **options):
"""
Test the root url of this site.
"""
username, key_name, zone, instance_ids = _read_server_list()
-
+ headers = options.get('headers', '')
+ csv_filename = options.get("csv_filename", '')
+
+ if csv_filename:
+ try:
+ stream = open(csv_filename, 'w')
+ except IOError, e:
+ raise IOError("Specified csv_filename='%s' is not writable. Check permissions or specify a different filename and try again." % csv_filename)
+
if not instance_ids:
print 'No bees are ready to attack.'
return
@@ -357,7 +410,7 @@ def attack(url, n, c, **options):
'num_requests': requests_per_instance,
'username': username,
'key_name': key_name,
- 'headers': options.get('headers', ''),
+ 'headers': headers,
'post_file': options.get('post_file'),
'mime_type': options.get('mime_type', ''),
})
@@ -379,6 +432,6 @@ def attack(url, n, c, **options):
print 'Offensive complete.'
- _print_results(results)
+ _print_results(results, params, csv_filename)
print 'The swarm is awaiting new orders.'
View
5 beeswithmachineguns/main.py
@@ -102,6 +102,9 @@ def parse_options():
attack_group.add_option('-H', '--headers', metavar="HEADERS", nargs=1,
action='store', dest='headers', type='string', default='',
help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2")
+ attack_group.add_option('-e', '--csv', metavar="FILENAME", nargs=1,
+ action='store', dest='csv_filename', type='string', default='',
+ help="Store the distribution of results in a csv file for all completed bees (default: '').")
parser.add_option_group(attack_group)
@@ -135,9 +138,11 @@ def parse_options():
headers=options.headers,
post_file=options.post_file,
mime_type=options.mime_type,
+ csv_filename=options.csv_filename,
)
bees.attack(options.url, options.number, options.concurrent, **additional_options)
+
elif command == 'down':
bees.down()
elif command == 'report':
Something went wrong with that request. Please try again.