Merge pull request twitter#3 from twitter/master

merge twitter twemproxy master changes

.gitignore

@@ -1,3 +1,6 @@
+# pyc
+*.pyc
+
 # Compiled Object files
 *.lo
 *.o

ChangeLog

@@ -1,3 +1,13 @@
+2014-18-10  idning <idning@gmail.com>
+
+    * twemproxy: version 0.4.0 release
+      mget improve (idning)
+      many new commands supported: LEX, PFADD, PFMERGE, SORT, PING, QUIT, SCAN... (mattrobenolt, areina, idning)
+      handle max open file limit(allenlz)
+      add notice-log and use ms time in log(idning)
+      fix bug in string_compare (andyqzb)
+      fix deadlock in sighandler (idning)
+
 2013-20-12  Manju Rajashekhar  <manj@cs.stanford.edu>
     * twemproxy: version 0.3.0 release
       SRANDMEMBER support for the optional count argument (mkhq)

configure.ac

@@ -1,6 +1,6 @@
 # Define the package version numbers and the bug reporting address
 m4_define([NC_MAJOR], 0)
-m4_define([NC_MINOR], 3)
+m4_define([NC_MINOR], 4)
 m4_define([NC_PATCH], 0)
 m4_define([NC_BUGS], [manj@cs.stanford.edu])
 

notes/redis.md

@@ -39,7 +39,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      RESTORE      |    Yes     | RESTORE key ttl serialized-value                                                                                    |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      SORT         |    No      | SORT key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]   |
+    |      SORT         |    Yes*    | SORT key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]   |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |       TTL         |    Yes     | TTL key                                                                                                             |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
@@ -48,6 +48,8 @@
     |      SCAN         |    No      | SCAN cursor [MATCH pattern] [COUNT count]                                                                           |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
+* SORT support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command.
+
 ### Strings Command
 
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
@@ -79,7 +81,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      MGET         |    Yes     | MGET key [key ...]                                                                                                  |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      MSET         |    No      | MSET key value [key value ...]                                                                                      |
+    |      MSET         |    Yes*    | MSET key value [key value ...]                                                                                      | 
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      MSETNX       |    No      | MSETNX key value [key value ...]                                                                                    |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
@@ -98,6 +100,8 @@
     |      STRLEN       |    Yes     | STRLEN key                                                                                                          |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
+* MSET support is not Atomic 
+
 ### Hashes
 
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
@@ -129,7 +133,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |      HVALS        |    Yes     | HVALS key                                                                                                           |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      HSCAN        |    No      | HSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
+    |      HSCAN        |    Yes     | HSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
 ### Lists
@@ -207,7 +211,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |   SUNIONSTORE     |    Yes*    | SUNIONSTORE destination key [key ...]                                                                               |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      SSCAN        |    No      | SSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
+    |      SSCAN        |    Yes     | SSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
 * SIDFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION and SUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to.
@@ -256,7 +260,7 @@
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
     |    ZUNIONSTORE    |    Yes*    | ZUNIONSTORE destination numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM|MIN|MAX]                 |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
-    |      ZSCAN        |    No      | ZSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
+    |      ZSCAN        |    Yes     | ZSCAN key cursor [MATCH pattern] [COUNT count]                                                                      |
     +-------------------+------------+---------------------------------------------------------------------------------------------------------------------+
 
 * ZINTERSTORE and ZUNIONSTORE support requires that the supplied keys hash to the same server. You can ensure this by using the same [hashtag](notes/recommendation.md#hash-tags) for all keys in the command. Twemproxy does no checking on its end to verify that all the keys hash to the same server, and the given command is forwarded to the server that the first key hashes to.

scripts/benchmark-mget.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+#coding: utf-8
+#file   : test_mget.py
+#author : ning
+#date   : 2014-04-01 13:15:48
+
+import os
+import re
+import commands
+
+ports = [
+    4001,  # before improve
+    4000,  # after improve
+    2000   # redis
+]
+
+def system(cmd):
+    return commands.getoutput(cmd)
+
+def extra(regex, text):
+    match = re.search(regex, text, re.DOTALL)
+    if match:
+        return match.group(1)
+
+def testit():
+    for mget_size in [10, 100, 1000, 10000]:
+        for port in ports:
+            cnt = 100*1000 / mget_size
+            clients = 50
+            if mget_size == 10000:
+                clients = 2
+            cmd = 'cd /home/ning/xredis/deploy-srcs/redis-2.8.3/src && ./redis-benchmark.%d -n %d -p %d -t mget  -r 1000000000 -c %d' % (mget_size, cnt, port, clients)
+            #print cmd
+            rst = system(cmd)
+
+            #100.00% <= 2 milliseconds
+            #28089.89 requests per second
+            rtime = extra('100.00% <= (\d+) milliseconds', rst)
+            qps = extra('([\.\d]+) requests per second', rst)
+
+            print 'mget_size=%d on %d: pqs: %s, rtime: %s' % (mget_size, port, qps, rtime)
+
+testit()

src/nc_core.c

@@ -208,9 +208,9 @@ core_send(struct context *ctx, struct conn *conn)
 
     status = conn->send(ctx, conn);
     if (status != NC_OK) {
-        log_debug(LOG_INFO, "send on %c %d failed: %s",
+        log_debug(LOG_INFO, "send on %c %d failed: status: %d errno: %d %s",
                   conn->client ? 'c' : (conn->proxy ? 'p' : 's'), conn->sd,
-                  strerror(errno));
+                  status, errno, strerror(errno));
     }
 
     return status;

src/nc_core.h

@@ -90,6 +90,8 @@ struct event_base;
 #include <stdbool.h>
 #include <inttypes.h>
 #include <string.h>
+#include <stdio.h>
+#include <ctype.h>
 #include <errno.h>
 #include <limits.h>
 #include <time.h>
@@ -114,6 +116,7 @@ struct event_base;
 #include <nc_mbuf.h>
 #include <nc_message.h>
 #include <nc_connection.h>
+#include <nc_server.h>
 
 struct context {
     uint32_t           id;          /* unique context id */

src/nc_log.c

@@ -133,11 +133,10 @@ _log(const char *file, int line, int panic, const char *fmt, ...)
 {
     struct logger *l = &logger;
     int len, size, errno_save;
-    char buf[LOG_MAX_LEN], *timestr;
+    char buf[LOG_MAX_LEN];
     va_list args;
-    struct tm *local;
-    time_t t;
     ssize_t n;
+    struct timeval tv;
 
     if (l->fd < 0) {
         return;
@@ -147,12 +146,11 @@ _log(const char *file, int line, int panic, const char *fmt, ...)
     len = 0;            /* length of output buffer */
     size = LOG_MAX_LEN; /* size of output buffer */
 
-    t = time(NULL);
-    local = localtime(&t);
-    timestr = asctime(local);
-
-    len += nc_scnprintf(buf + len, size - len, "[%.*s] %s:%d ",
-                        strlen(timestr) - 1, timestr, file, line);
+    gettimeofday(&tv, NULL);
+    buf[len++] = '[';
+    len += nc_strftime(buf + len, size - len, "%Y-%m-%d %H:%M:%S.", localtime(&tv.tv_sec));
+    len += nc_scnprintf(buf + len, size - len, "%03ld", tv.tv_usec/1000);
+    len += nc_scnprintf(buf + len, size - len, "] %s:%d ", file, line);
 
     va_start(args, fmt);
     len += nc_vscnprintf(buf + len, size - len, fmt, args);
@@ -261,6 +259,13 @@ _log_hexdump(const char *file, int line, char *data, int datalen,
         l->nerror++;
     }
 
+    if (len >= size - 1) {
+        n = nc_write(l->fd, "\n", 1);
+        if (n < 0) {
+            l->nerror++;
+        }
+    }
+
     errno = errno_save;
 }
 
@@ -281,7 +286,7 @@ _log_safe(const char *fmt, ...)
     len = 0;            /* length of output buffer */
     size = LOG_MAX_LEN; /* size of output buffer */
 
-    len += nc_safe_snprintf(buf + len, size - len, "[........................] ");
+    len += nc_safe_snprintf(buf + len, size - len, "[.......................] ");
 
     va_start(args, fmt);
     len += nc_safe_vsnprintf(buf + len, size - len, fmt, args);
@@ -310,7 +315,7 @@ _log_stderr_safe(const char *fmt, ...)
     len = 0;            /* length of output buffer */
     size = LOG_MAX_LEN; /* size of output buffer */
 
-    len += nc_safe_snprintf(buf + len, size - len, "[........................] ");
+    len += nc_safe_snprintf(buf + len, size - len, "[.......................] ");
 
     va_start(args, fmt);
     len += nc_safe_vsnprintf(buf + len, size - len, fmt, args);