Merge "Redis API in Scylla"

Merged patch series from Peng Jian, adding optionally-enabled Redis API
support to Scylla. This feature is experimental, and partial - the extent
of this support is detailed in docs/redis/redis.md.

Patches:
   Document: add docs/redis/redis.md
   redis: Redis API in Scylla
   Redis API: graft redis module to Scylla
   redis-test: add test cases for Redis API

configure.py

@@ -456,6 +456,8 @@ def find_headers(repodir, excluded_dirs):
                         help='enable allocation failure injection')
 arg_parser.add_argument('--with-antlr3', dest='antlr3_exec', action='store', default=None,
                         help='path to antlr3 executable')
+arg_parser.add_argument('--with-ragel', dest='ragel_exec', action='store', default='ragel',
+        help='path to ragel executable')
 args = arg_parser.parse_args()
 
 defines = ['XXH_PRIVATE_API',
@@ -802,6 +804,26 @@ def find_headers(repodir, excluded_dirs):
        'alternator/auth.cc',
 ]
 
+redis = [
+        'redis/service.cc',
+        'redis/server.cc',
+        'redis/query_processor.cc',
+        'redis/protocol_parser.rl',
+        'redis/keyspace_utils.cc',
+        'redis/options.cc',
+        'redis/stats.cc',
+        'redis/mutation_utils.cc',
+        'redis/query_utils.cc',
+        'redis/abstract_command.cc',
+        'redis/command_factory.cc',
+        'redis/commands/unknown.cc',
+        'redis/commands/ping.cc',
+        'redis/commands/set.cc',
+        'redis/commands/get.cc',
+        'redis/commands/del.cc',
+        'redis/commands/select.cc',
+        ]
+
 idls = ['idl/gossip_digest.idl.hh',
         'idl/uuid.idl.hh',
         'idl/range.idl.hh',
@@ -847,7 +869,7 @@ def find_headers(repodir, excluded_dirs):
 ]
 
 deps = {
-    'scylla': idls + ['main.cc', 'release.cc'] + scylla_core + api + alternator,
+    'scylla': idls + ['main.cc', 'release.cc'] + scylla_core + api + alternator + redis,
 }
 
 pure_boost_tests = set([
@@ -1237,6 +1259,11 @@ def configure_zstd(build_dir, mode):
 else:
     antlr3_exec = "antlr3"
 
+if args.ragel_exec:
+    ragel_exec = args.ragel_exec
+else:
+    ragel_exec = "ragel"
+
 for mode in build_modes:
     configure_zstd(outdir, mode)
 
@@ -1271,6 +1298,11 @@ def configure_zstd(build_dir, mode):
             command = {ninja} -C $subdir $target
             restat = 1
             description = NINJA $out
+        rule ragel
+            # sed away a bug in ragel 7 that emits some extraneous _nfa* variables
+            # (the $$ is collapsed to a single one by ninja)
+            command = {ragel_exec} -G2 -o $out $in && sed -i -e '1h;2,$$H;$$!d;g' -re 's/static const char _nfa[^;]*;//g' $out
+            description = RAGEL $out
         rule run
             command = $in > $out
             description = GEN $out
@@ -1335,6 +1367,7 @@ def configure_zstd(build_dir, mode):
         swaggers = {}
         serializers = {}
         thrifts = set()
+        ragels = {}
         antlr3_grammars = set()
         seastar_dep = 'build/{}/seastar/libseastar.a'.format(mode)
         for binary in build_artifacts:
@@ -1392,6 +1425,9 @@ def configure_zstd(build_dir, mode):
                 elif src.endswith('.json'):
                     hh = '$builddir/' + mode + '/gen/' + src + '.hh'
                     swaggers[hh] = src
+                elif src.endswith('.rl'):
+                    hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh')
+                    ragels[hh] = src
                 elif src.endswith('.thrift'):
                     thrifts.add(src)
                 elif src.endswith('.g'):
@@ -1420,6 +1456,7 @@ def configure_zstd(build_dir, mode):
             gen_headers += g.headers('$builddir/{}/gen'.format(mode))
         gen_headers += list(swaggers.keys())
         gen_headers += list(serializers.keys())
+        gen_headers += list(ragels.keys())
         gen_headers_dep = ' '.join(gen_headers)
 
         for obj in compiles:
@@ -1433,6 +1470,9 @@ def configure_zstd(build_dir, mode):
         for hh in serializers:
             src = serializers[hh]
             f.write('build {}: serializer {} | idl-compiler.py\n'.format(hh, src))
+        for hh in ragels:
+            src = ragels[hh]
+            f.write('build {}: ragel {}\n'.format(hh, src))
         for thrift in thrifts:
             outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode)))
             f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source))

db/config.cc

@@ -713,6 +713,21 @@ db::config::config(std::shared_ptr<db::extensions> exts)
     , alternator_address(this, "alternator_address", value_status::Used, "0.0.0.0", "Alternator API listening address")
     , alternator_enforce_authorization(this, "alternator_enforce_authorization", value_status::Used, false, "Enforce checking the authorization header for every request in Alternator")
     , abort_on_ebadf(this, "abort_on_ebadf", value_status::Used, true, "Abort the server on incorrect file descriptor access. Throws exception when disabled.")
+    , redis_transport_port(this, "redis_transport_port", value_status::Used, 6379, "Port on which the REDIS transport listens for clients.")
+    // FIXME: 9142 has been used by native_transport_port_ssl
+    , redis_transport_port_ssl(this, "redis_transport_port_ssl", value_status::Used, 9142, "Port on which the REDIS TLS native transport listens for clients.")
+    , enable_redis_protocol(this, "enable_redis_protocol", value_status::Used, false, "Enable redis protocol; Scylla will process redis protocol as a redis cluster if enable.")
+    , redis_read_consistency_level(this, "redis_read_consistency_level", value_status::Used, "ONE", "Consistency level for read operations for redis.")
+    , redis_write_consistency_level(this, "redis_write_consistency_level", value_status::Used, "ANY", "Consistency level for write operations for redis.")
+    , redis_default_database_count(this, "redis_default_database_count", value_status::Used, 16, "Default database count for the redis cluster.")
+    , redis_keyspace_options(this, "redis_keyspace_options", value_status::Used, {}, 
+            "Enable redis protocol, list of properties of replication of redis keyspace. The available options are:\n"
+            "\n"
+            "\tclass : (Default: SimpleStrategy ).\n"
+            "\treplication_factor: (Default: 1) If the class is SimpleStrategy, set the replication factor for the strategy.\n"
+            "\tdatacenter_i: (Default: 1) If the class is NetworkTopologyStrategy, set the replication factor per datacenter.\n"
+            "\n"
+            "The properties of replication for redis keyspace.")
 
     , default_log_level(this, "default_log_level", value_status::Used)
     , logger_log_level(this, "logger_log_level", value_status::Used)

db/config.hh

@@ -297,6 +297,14 @@ public:
     named_value<bool> alternator_enforce_authorization;
     named_value<bool> abort_on_ebadf;
 
+    named_value<uint16_t> redis_transport_port;
+    named_value<uint16_t> redis_transport_port_ssl;
+    named_value<bool> enable_redis_protocol;
+    named_value<sstring> redis_read_consistency_level;
+    named_value<sstring> redis_write_consistency_level;
+    named_value<uint16_t> redis_default_database_count;
+    named_value<string_map> redis_keyspace_options;
+
     seastar::logging_settings logging_settings(const boost::program_options::variables_map&) const;
 
     boost::program_options::options_description_easy_init&