Skip to content
Find file
Fetching contributors…
Cannot retrieve contributors at this time
605 lines (540 sloc) 20.2 KB
/* Redis load utility.
*
* Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* This software is NOT released under a free software license.
* It is a commercial tool, under the terms of the license you can find in
* the COPYING file in the Redis-Tools distribution.
*/
#include "fmacros.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>
#include <errno.h>
#include <signal.h>
#include <assert.h>
#include <signal.h>
#include <math.h>
#include <limits.h>
#include "hiredis.h"
#include "adapters/ae.h"
#include "adlist.h"
#include "zmalloc.h"
#include "rc4rand.h"
#include "utils.h"
#define REDIS_IDLE 0
#define REDIS_GET 1
#define REDIS_SET 2
#define REDIS_DEL 3
#define REDIS_SWAPIN 4
#define REDIS_LPUSH 6
#define REDIS_LPOP 7
#define REDIS_HSET 8
#define REDIS_HGET 9
#define REDIS_HGETALL 10
#define MAX_LATENCY 5000
#define DEFAULT_KEYSPACE 100000 /* 100k */
#define DEFAULT_HASHKEYSPACE 1000 /* 1k */
#define REDIS_NOTUSED(V) ((void) V)
static struct config {
aeEventLoop *el;
int debug;
int done;
list *clients;
int num_clients;
int num_requests;
int issued_requests;
int datasize_min;
int datasize_max;
unsigned char *databuf;
int keyspace;
int hashkeyspace;
int set_perc;
int del_perc;
int swapin_perc;
int lpush_perc;
int lpop_perc;
int hset_perc;
int hget_perc;
int hgetall_perc;
int check;
int rand;
int longtail;
int longtail_order;
char *hostip;
int hostport;
int keepalive;
long long start;
long long totlatency;
int *latency;
int quiet;
int loop;
int idlemode;
int ctrlc; /* Ctrl + C pressed */
unsigned int prngseed;
/* The following "optab" array is used in order to randomize the different
* kind of operations, like GET, SET, LPUSH, LPOP, SADD, and so forth.
* For every query the client will pick a random bucket from 0 to 99
* and will check what is the kind of operation to perform, also it will
* use the bucket ID in order to make sure this kind of operation is
* always executed against the right data type, so for instance if bucket
* 7 is a LPUSH the operation will be performed against key xxxxxx07
* and so forth. */
unsigned char optab[100];
} config;
typedef struct _client {
redisAsyncContext *context;
int state;
int reqtype; /* request type. REDIS_GET, REDIS_SET, ... */
long long start; /* start time in milliseconds */
long keyid; /* the key name for this request is "key:<keyid>" */
} *client;
/* Prototypes */
static void issueRequest(client c);
static void createMissingClients(void);
/* Return a pseudo random number between min and max both inclusive */
static long randbetween(long min, long max) {
return min+(random()%(max-min+1));
}
/* PRNG biased accordingly to the power law (Long Tail alike) */
unsigned long longtailprng(unsigned long min, unsigned long max, int n) {
unsigned long pl;
double r = (double)(random()&(INT_MAX-1))/INT_MAX;
max += 1;
pl = pow((pow(max,n+1) - pow(min,n+1))*r + pow(min,n+1),1.0/(n+1));
return (max-1-pl)+min;
}
static void clientDisconnected(const redisAsyncContext *context, int status) {
listNode *ln;
client c = (client)context->data;
if (status != REDIS_OK) {
fprintf(stderr,"Disconnected: %s\n",c->context->errstr);
exit(1);
}
ln = listSearchKey(config.clients,c);
assert(ln != NULL);
listDelNode(config.clients,ln);
zfree(c);
/* The run was not done, create new client(s). */
if (!config.done) {
createMissingClients();
}
/* Stop the event loop when all clients were disconnected */
if (!listLength(config.clients)) {
aeStop(config.el);
}
}
static client createClient(void) {
client c = zmalloc(sizeof(struct _client));
c->context = redisAsyncConnect(config.hostip,config.hostport);
c->context->data = c;
redisAsyncSetDisconnectCallback(c->context,clientDisconnected);
if (c->context->err) {
fprintf(stderr,"Connect: %s\n",c->context->errstr);
exit(1);
}
redisAeAttach(config.el,c->context);
listAddNodeTail(config.clients,c);
issueRequest(c);
return c;
}
static void createMissingClients(void) {
while(listLength(config.clients) < (size_t)config.num_clients) {
createClient();
}
}
static void checkDataIntegrity(client c, redisReply *reply) {
if (c->reqtype == REDIS_GET && reply->type == REDIS_REPLY_STRING) {
unsigned char *data;
unsigned int datalen;
rc4rand_seed(c->keyid);
datalen = rc4rand_between(config.datasize_min,config.datasize_max);
data = zmalloc(datalen);
rc4rand_set(data,datalen);
if (reply->len != (int)datalen) {
fprintf(stderr, "*** Len mismatch for KEY key:%ld\n", c->keyid);
fprintf(stderr, "*** %d instead of %d\n", reply->len, datalen);
fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
exit(1);
}
if (memcmp(reply->str,data,datalen) != 0) {
fprintf(stderr, "*** Data mismatch for KEY key:%ld\n", c->keyid);
fprintf(stderr, "*** '%s' instead of '%s'\n", reply->str, data);
exit(1);
}
zfree(data);
}
}
static void handleReply(redisAsyncContext *context, void *_reply, void *privdata) {
REDIS_NOTUSED(privdata);
redisReply *reply = (redisReply*)_reply;
client c = (client)context->data;
long long latency = (microseconds() - c->start) / 1000;
if (reply == NULL && context->err) {
fprintf(stderr,"Error: %s\n", context->errstr);
exit(1);
} else {
assert(reply != NULL);
if (reply->type == REDIS_REPLY_ERROR) {
fprintf(stderr,"Error: %s\n", reply->str);
exit(1);
}
}
if (latency > MAX_LATENCY) latency = MAX_LATENCY;
config.latency[latency]++;
if (config.check) checkDataIntegrity(c,reply);
freeReplyObject(reply);
if (config.done || config.ctrlc) {
redisAsyncDisconnect(c->context);
return;
}
if (config.keepalive) {
issueRequest(c);
} else {
/* createMissingClients will be called in the disconnection callback */
redisAsyncDisconnect(c->context);
}
}
static unsigned long randomData(long seed) {
unsigned long datalen;
/* We use the key number as seed of the PRNG, so we'll be able to check if
* a given key contains the right data later, without the use of additional
* memory. */
if (config.check) {
rc4rand_seed(seed);
datalen = rc4rand_between(config.datasize_min,config.datasize_max);
rc4rand_set(config.databuf,datalen);
} else {
datalen = randbetween(config.datasize_min,config.datasize_max);
if (config.rand) {
rc4rand_seed(seed);
rc4rand_set(config.databuf,datalen);
} else {
memset(config.databuf,'x',datalen);
}
}
return datalen;
}
static void issueRequest(client c) {
int op = config.optab[random() % 100];
long key, hashkey;
unsigned long datalen;
config.issued_requests++;
if (config.issued_requests == config.num_requests) config.done = 1;
c->start = microseconds();
if (config.longtail) {
key = longtailprng(0,config.keyspace-1,config.longtail_order);
hashkey = longtailprng(0,config.hashkeyspace-1,config.longtail_order);
} else {
key = random() % config.keyspace;
hashkey = random() % config.hashkeyspace;
}
c->keyid = key;
c->reqtype = op;
if (op == REDIS_IDLE) {
/* Idle */
} else if (op == REDIS_SET) {
datalen = randomData(key);
redisAsyncCommand(c->context,handleReply,NULL,"SET string:%ld %b",key,config.databuf,datalen);
} else if (op == REDIS_GET) {
redisAsyncCommand(c->context,handleReply,NULL,"GET string:%ld",key);
} else if (op == REDIS_DEL) {
redisAsyncCommand(c->context,handleReply,NULL,"DEL string:%ld list:%ld hash:%ld",key,key,key);
} else if (op == REDIS_LPUSH) {
datalen = randomData(key);
redisAsyncCommand(c->context,handleReply,NULL,"LPUSH list:%ld %b",key,config.databuf,datalen);
} else if (op == REDIS_LPOP) {
redisAsyncCommand(c->context,handleReply,NULL,"LPOP list:%ld",key);
} else if (op == REDIS_HSET) {
datalen = randomData(key);
redisAsyncCommand(c->context,handleReply,NULL,"HSET hash:%ld key:%ld %b",key,hashkey,config.databuf,datalen);
} else if (op == REDIS_HGET) {
redisAsyncCommand(c->context,handleReply,NULL,"HGET hash:%ld key:%ld",key,hashkey);
} else if (op == REDIS_HGETALL) {
redisAsyncCommand(c->context,handleReply,NULL,"HGETALL hash:%ld",key);
} else if (op == REDIS_SWAPIN) {
/* Only accepts a single argument, so for now only works with string keys. */
redisAsyncCommand(c->context,handleReply,NULL,"DEBUG SWAPIN string:%ld",key);
} else {
assert(NULL);
}
}
static void showLatencyReport(void) {
int j, seen = 0;
float perc, reqpersec;
reqpersec = (float)config.issued_requests/((float)config.totlatency/1000);
if (!config.quiet) {
printf("====== Report ======\n");
printf(" %d requests in %.3f seconds\n", config.issued_requests,
(float)config.totlatency/1000);
printf(" %.2f requests per second\n", reqpersec);
printf(" %d parallel clients\n", config.num_clients);
printf(" payload: %d..%d bytes\n", config.datasize_min, config.datasize_max);
printf(" keep alive: %d\n", config.keepalive);
printf("\n");
for (j = 0; j <= MAX_LATENCY; j++) {
if (config.latency[j]) {
seen += config.latency[j];
perc = ((float)seen*100)/config.issued_requests;
printf("%6.2f%% < %d ms\n", perc, j+1);
}
}
} else {
printf("%.2f requests per second\n", reqpersec);
}
}
static void prepareForBenchmark(void) {
memset(config.latency,0,sizeof(int)*(MAX_LATENCY+1));
config.start = microseconds();
config.issued_requests = 0;
}
static void endBenchmark(void) {
config.totlatency = (microseconds()-config.start)/1000;
showLatencyReport();
}
static void usage(char *wrong) {
if (wrong)
printf("Wrong option '%s' or option argument missing\n\n",wrong);
printf(
"Usage: redis-load ... options ...\n\n"
" host <hostname> Server hostname (default 127.0.0.1)\n"
" port <hostname> Server port (default 6379)\n"
" clients <clients> Number of parallel connections (default 50)\n"
" requests <requests> Total number of requests (default 10k)\n"
" mindatasize <size> Min data size of string values in bytes (default 1)\n"
" maxdatasize <size> Min data size of string values in bytes (default 64)\n"
" datasize <size> Set both min and max data size to the same value\n"
" keepalive 1=keep alive 0=reconnect (default 1)\n"
" keyspace The number of different keys to use (default 100k)\n"
" rand Use random data payload (incompressible)\n"
" check Check integrity where reading data back (implies rand)\n"
" longtail Use long tail alike key access pattern distribution\n"
" longtailorder A value of 2: 20%% keys get 49%% accesses.\n"
" 3: 20%% keys get 59%% accesses.\n"
" 4: 20%% keys get 67%% accesses.\n"
" 5: 20%% keys get 74%% accesses.\n"
" 6: 20%% keys get 79%% accesses (default).\n"
" 7: 20%% keys get 83%% accesses.\n"
" 8: 20%% keys get 86%% accesses.\n"
" 9: 20%% keys get 89%% accesses.\n"
" 10: 20%% keys get 91%% accesses.\n"
" 20: 20%% keys get 99%% accesses.\n"
" seed <seed> PRNG seed for deterministic load\n"
" big alias for keyspace 1000000 requests 1000000\n"
" verybig alias for keyspace 10000000 requests 10000000\n"
" quiet Quiet mode, less verbose\n"
" loop Loop. Run the tests forever\n"
" idle Idle mode. Just open N idle connections and wait.\n"
" debug Debug mode. more verbose.\n"
"\n"
"Type of operations (use percentages without trailing %%):\n"
"\n"
" set <percentage> Percentage of SETs (default 50)\n"
" del <percentage> Percentage of DELs (default 0)\n"
" lpush <percentage> Percentage of LPUSHs (default 0)\n"
" lpop <percentage> Percentage of LPOPs (default 0)\n"
" hset <percentage> Percentage of HSETs (default 0)\n"
" hget <percentage> Percentage of HGETs (default 0)\n"
" hgetall <percentage> Percentage of HGETs (default 0)\n"
" swapin <percentage> Percentage of DEBUG SWAPINs (default 0)\n"
"\n"
" All the free percantege (in order to reach 100%%) will be used for GETs\n"
);
exit(1);
}
static void parseOptions(int argc, char **argv) {
int i;
for (i = 1; i < argc; i++) {
int lastarg = i==argc-1;
if (!strcmp(argv[i],"clients") && !lastarg) {
config.num_clients = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"requests") && !lastarg) {
config.num_requests = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"set") && !lastarg) {
config.set_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"del") && !lastarg) {
config.del_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"swapin") && !lastarg) {
config.swapin_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"lpush") && !lastarg) {
config.lpush_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"lpop") && !lastarg) {
config.lpop_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"hset") && !lastarg) {
config.hset_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"hget") && !lastarg) {
config.hget_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"hgetall") && !lastarg) {
config.hgetall_perc = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"keepalive") && !lastarg) {
config.keepalive = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"host") && !lastarg) {
config.hostip = argv[i+1];
i++;
} else if (!strcmp(argv[i],"port") && !lastarg) {
config.hostport = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"datasize") && !lastarg) {
config.datasize_max = config.datasize_min = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"mindatasize") && !lastarg) {
config.datasize_min = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"maxdatasize") && !lastarg) {
config.datasize_max = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"keyspace") && !lastarg) {
config.keyspace = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"hashkeyspace") && !lastarg) {
config.hashkeyspace = atoi(argv[i+1]);
i++;
} else if (!strcmp(argv[i],"seed") && !lastarg) {
config.prngseed = strtol(argv[i+1],NULL,10);
i++;
} else if (!strcmp(argv[i],"big")) {
config.keyspace = 1000000;
config.num_requests = 1000000;
} else if (!strcmp(argv[i],"verybig")) {
config.keyspace = 10000000;
config.num_requests = 10000000;
} else if (!strcmp(argv[i],"quiet")) {
config.quiet = 1;
} else if (!strcmp(argv[i],"check")) {
config.check = 1;
} else if (!strcmp(argv[i],"rand")) {
config.rand = 1;
} else if (!strcmp(argv[i],"longtail")) {
config.longtail = 1;
} else if (!strcmp(argv[i],"longtailorder") && !lastarg) {
config.longtail_order = atoi(argv[i+1]);
i++;
if (config.longtail_order < 2 || config.longtail_order > 100) {
printf("Value out of range for 'longtailorder' option");
exit(1);
}
} else if (!strcmp(argv[i],"loop")) {
config.loop = 1;
} else if (!strcmp(argv[i],"debug")) {
config.debug = 1;
} else if (!strcmp(argv[i],"idle")) {
config.idlemode = 1;
} else if (!strcmp(argv[i],"help")) {
usage(NULL);
} else {
usage(argv[i]);
}
}
/* Sanitize options */
if (config.datasize_min < 1) config.datasize_min = 1;
if (config.datasize_min > 1024*1024) config.datasize_min = 1024*1024;
if (config.datasize_max < 1) config.datasize_max = 1;
if (config.datasize_max > 1024*1024) config.datasize_max = 1024*1024;
if (config.keyspace < 1) config.keyspace = DEFAULT_KEYSPACE;
if (config.hashkeyspace < 1) config.hashkeyspace = DEFAULT_HASHKEYSPACE;
}
static void ctrlc(int sig) {
REDIS_NOTUSED(sig);
if (config.idlemode) {
exit(1);
} else {
config.ctrlc++;
if (config.ctrlc == 1) {
config.done = 1;
printf("\nWaiting for pending requests to complete...\n");
} else {
printf("\nForcing exit...\n");
exit(1);
}
}
}
static void fillOpTab(int *i, int op, int perc) {
int j;
for (j = 0; j < perc; j++) {
if (*i < 100) {
config.optab[*i] = op;
(*i)++;
}
}
}
int main(int argc, char **argv) {
int i = 0;
signal(SIGHUP, SIG_IGN);
signal(SIGPIPE, SIG_IGN);
config.el = aeCreateEventLoop();
config.debug = 0;
config.done = 0;
config.clients = listCreate();
config.num_clients = 50;
config.num_requests = 10000;
config.issued_requests = 0;
config.keepalive = 1;
config.set_perc = 50;
config.del_perc = 0;
config.swapin_perc = 0;
config.lpush_perc = 0;
config.lpop_perc = 0;
config.hset_perc = 0;
config.hget_perc = 0;
config.hgetall_perc = 0;
config.datasize_min = 1;
config.datasize_max = 64;
config.keyspace = DEFAULT_KEYSPACE; /* 100k */
config.hashkeyspace = DEFAULT_HASHKEYSPACE; /* 1k */
config.check = 0;
config.rand = 0;
config.longtail = 0;
config.quiet = 0;
config.loop = 0;
config.idlemode = 0;
config.latency = NULL;
config.latency = zmalloc(sizeof(int)*(MAX_LATENCY+1));
config.ctrlc = 0;
config.prngseed = (unsigned int) (microseconds()^getpid());
config.hostip = "127.0.0.1";
config.hostport = 6379;
parseOptions(argc,argv);
config.databuf = zmalloc(config.datasize_max);
if (config.keepalive == 0) {
printf("WARNING: keepalive disabled, you probably need 'echo 1 > /proc/sys/net/ipv4/tcp_tw_reuse' for Linux and 'sudo sysctl -w net.inet.tcp.msl=1000' for Mac OS X in order to use a lot of clients/requests\n");
}
if (config.idlemode) {
printf("Creating %d idle connections and waiting forever (Ctrl+C when done)\n", config.num_clients);
memset(config.optab,REDIS_IDLE,100);
} else {
/* Setup the operation table. Start with a table with just GET
* operations and overwrite it with others as needed. */
memset(config.optab,REDIS_GET,100);
fillOpTab(&i,REDIS_SET,config.set_perc);
fillOpTab(&i,REDIS_DEL,config.del_perc);
fillOpTab(&i,REDIS_LPUSH,config.lpush_perc);
fillOpTab(&i,REDIS_LPOP,config.lpop_perc);
fillOpTab(&i,REDIS_HSET,config.hset_perc);
fillOpTab(&i,REDIS_HGET,config.hget_perc);
fillOpTab(&i,REDIS_HGETALL,config.hgetall_perc);
fillOpTab(&i,REDIS_SWAPIN,config.swapin_perc);
}
signal(SIGINT,ctrlc);
srandom(config.prngseed);
printf("PRNG seed is: %u - use the 'seed' option to reproduce the same sequence\n", config.prngseed);
do {
prepareForBenchmark();
createMissingClients();
aeMain(config.el);
endBenchmark();
} while(config.loop);
return 0;
}
Something went wrong with that request. Please try again.