Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

redis-cli --pipe for mass import.

  • Loading branch information...
commit f1af84d7143636602555e5d2d949bd5c9c6f6a69 1 parent 59b084b
@antirez authored
Showing with 143 additions and 3 deletions.
  1. +1 −1  src/Makefile
  2. +142 −2 src/redis-cli.c
View
2  src/Makefile
@@ -60,7 +60,7 @@ INSTALL= cp -pf
OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o vm.o pubsub.o multi.o debug.o sort.o intset.o syncio.o slowlog.o bio.o memtest.o
BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o
-CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o
+CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o ae.o
CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o
CHECKAOFOBJ = redis-check-aof.o
View
144 src/redis-cli.c
@@ -46,6 +46,8 @@
#include "zmalloc.h"
#include "linenoise.h"
#include "help.h"
+#include "anet.h"
+#include "ae.h"
#define REDIS_NOTUSED(V) ((void) V)
@@ -67,6 +69,7 @@ static struct config {
int pubsub_mode;
int latency_mode;
int slave_mode;
+ int pipe_mode;
int bigkeys;
int stdinarg; /* get last arg from stdin. (-x option) */
char *auth;
@@ -621,6 +624,8 @@ static int parseOptions(int argc, char **argv) {
config.latency_mode = 1;
} else if (!strcmp(argv[i],"--slave")) {
config.slave_mode = 1;
+ } else if (!strcmp(argv[i],"--pipe")) {
+ config.pipe_mode = 1;
} else if (!strcmp(argv[i],"--bigkeys")) {
config.bigkeys = 1;
} else if (!strcmp(argv[i],"-d") && !lastarg) {
@@ -674,6 +679,7 @@ static void usage() {
" --raw Use raw formatting for replies (default when STDOUT is not a tty)\n"
" --latency Enter a special mode continuously sampling latency.\n"
" --slave Simulate a slave showing commands received from the master.\n"
+" --pipe Transfer raw Redis protocol from stdin to server.\n"
" --bigkeys Sample Redis keys looking for big keys.\n"
" --help Output this help and exit\n"
" --version Output version and exit\n"
@@ -878,6 +884,133 @@ static void slaveMode(void) {
while (cliReadReply(0) == REDIS_OK);
}
+static void pipeMode(void) {
+ int fd = context->fd;
+ long long errors = 0, replies = 0, obuf_len = 0, obuf_pos = 0;
+ char ibuf[1024*16], obuf[1024*16]; /* Input and output buffers */
+ char aneterr[ANET_ERR_LEN];
+ redisReader *reader = redisReaderCreate();
+ redisReply *reply;
+ int eof = 0; /* True once we consumed all the standard input. */
+ int done = 0;
+ char magic[20]; /* Special reply we recognize. */
+
+ srand(time(NULL));
+
+ /* Use non blocking I/O. */
+ if (anetNonBlock(aneterr,fd) == ANET_ERR) {
+ fprintf(stderr, "Can't set the socket in non blocking mode: %s\n",
+ aneterr);
+ exit(1);
+ }
+
+ /* Transfer raw protocol and read replies from the server at the same
+ * time. */
+ while(!done) {
+ int mask = AE_READABLE;
+
+ if (!eof || obuf_len != 0) mask |= AE_WRITABLE;
+ mask = aeWait(fd,mask,1000);
+
+ /* Handle the readable state: we can read replies from the server. */
+ if (mask & AE_READABLE) {
+ ssize_t nread;
+
+ /* Read from socket and feed the hiredis reader. */
+ do {
+ nread = read(fd,ibuf,sizeof(ibuf));
+ if (nread == -1 && errno != EAGAIN) {
+ fprintf(stderr, "Error reading from the server: %s\n",
+ strerror(errno));
+ exit(1);
+ }
+ if (nread > 0) redisReaderFeed(reader,ibuf,nread);
+ } while(nread > 0);
+
+ /* Consume replies. */
+ do {
+ if (redisReaderGetReply(reader,(void**)&reply) == REDIS_ERR) {
+ fprintf(stderr, "Error reading replies from server\n");
+ exit(1);
+ }
+ if (reply) {
+ if (reply->type == REDIS_REPLY_ERROR) {
+ fprintf(stderr,"%s\n", reply->str);
+ errors++;
+ } else if (eof && reply->type == REDIS_REPLY_STRING &&
+ reply->len == 20) {
+ /* Check if this is the reply to our final ECHO
+ * command. If so everything was received
+ * from the server. */
+ if (memcmp(reply->str,magic,20) == 0) {
+ printf("Last reply received from server.\n");
+ done = 1;
+ replies--;
+ }
+ }
+ replies++;
+ freeReplyObject(reply);
+ }
+ } while(reply);
+ }
+
+ /* Handle the writable state: we can send protocol to the server. */
+ if (mask & AE_WRITABLE) {
+ while(1) {
+ /* Transfer current buffer to server. */
+ if (obuf_len != 0) {
+ ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len);
+
+ if (nwritten == -1) {
+ fprintf(stderr, "Error writing to the server: %s\n",
+ strerror(errno));
+ exit(1);
+ }
+ obuf_len -= nwritten;
+ obuf_pos += nwritten;
+ if (obuf_len != 0) break; /* Can't accept more data. */
+ }
+ /* If buffer is empty, load from stdin. */
+ if (obuf_len == 0 && !eof) {
+ ssize_t nread = read(STDIN_FILENO,obuf,sizeof(obuf));
+
+ if (nread == 0) {
+ char echo[] =
+ "*2\r\n$4\r\nECHO\r\n$20\r\n01234567890123456789\r\n";
+ int j;
+
+ eof = 1;
+ /* Everything transfered, so we queue a special
+ * ECHO command that we can match in the replies
+ * to make sure everything was read from the server. */
+ for (j = 0; j < 20; j++)
+ magic[j] = rand() & 0xff;
+ memcpy(echo+19,magic,20);
+ memcpy(obuf,echo,sizeof(echo)-1);
+ obuf_len = sizeof(echo)-1;
+ obuf_pos = 0;
+ printf("All data transferred. Waiting for the last reply...\n");
+ } else if (nread == -1) {
+ fprintf(stderr, "Error reading from stdin: %s\n",
+ strerror(errno));
+ exit(1);
+ } else {
+ obuf_len = nread;
+ obuf_pos = 0;
+ }
+ }
+ if (obuf_len == 0 && eof) break;
+ }
+ }
+ }
+ redisReaderFree(reader);
+ printf("errors: %lld, replies: %lld\n", errors, replies);
+ if (errors)
+ exit(1);
+ else
+ exit(0);
+}
+
#define TYPE_STRING 0
#define TYPE_LIST 1
#define TYPE_SET 2
@@ -974,6 +1107,7 @@ int main(int argc, char **argv) {
config.pubsub_mode = 0;
config.latency_mode = 0;
config.slave_mode = 0;
+ config.pipe_mode = 0;
config.bigkeys = 0;
config.stdinarg = 0;
config.auth = NULL;
@@ -988,18 +1122,24 @@ int main(int argc, char **argv) {
argc -= firstarg;
argv += firstarg;
- /* Start in latency mode if appropriate */
+ /* Latency mode */
if (config.latency_mode) {
cliConnect(0);
latencyMode();
}
- /* Start in slave mode if appropriate */
+ /* Slave mode */
if (config.slave_mode) {
cliConnect(0);
slaveMode();
}
+ /* Pipe mode */
+ if (config.pipe_mode) {
+ cliConnect(0);
+ pipeMode();
+ }
+
/* Find big keys */
if (config.bigkeys) {
cliConnect(0);
Please sign in to comment.
Something went wrong with that request. Please try again.