From c60996ecbb956c3e86223e00be181b4cfbd67f53 Mon Sep 17 00:00:00 2001 From: bigbes Date: Tue, 9 Aug 2016 22:28:06 +0300 Subject: [PATCH] Adding SASL authentication + add documentation closes gh-4 + cmd_set isn't invoked in text protocol closes gh-24 --- CMakeLists.txt | 1 + README.md | 55 +- cmake/FindCyrusSASL.cmake | 42 ++ debian/control | 3 +- memcached/CMakeLists.txt | 5 + memcached/init.lua | 67 ++- memcached/internal/constants.h | 6 + memcached/internal/expiration.c | 18 +- memcached/internal/mc_sasl.c | 270 +++++++++ memcached/internal/mc_sasl.h | 28 + memcached/internal/memcached.c | 43 +- memcached/internal/memcached.h | 6 +- memcached/internal/proto_bin.c | 249 +++++--- memcached/internal/proto_txt.c | 2 + rpm/tarantool-memcached.spec | 1 + test-run | 2 +- test.sh | 2 +- test/.tarantoolctl | 8 +- test/internal/memcached_connection.py | 691 +++++++++------------- test/sasl/binary-sasl.result | 1 + test/sasl/binary-sasl.test.py | 148 +++++ test/sasl/config/tarantool-memcached.conf | 3 + test/sasl/internal | 1 + test/sasl/sasl.lua | 83 +++ test/sasl/suite.ini | 4 + 25 files changed, 1212 insertions(+), 527 deletions(-) create mode 100644 cmake/FindCyrusSASL.cmake create mode 100644 memcached/internal/mc_sasl.c create mode 100644 memcached/internal/mc_sasl.h create mode 100644 test/sasl/binary-sasl.result create mode 100644 test/sasl/binary-sasl.test.py create mode 100644 test/sasl/config/tarantool-memcached.conf create mode 120000 test/sasl/internal create mode 100644 test/sasl/sasl.lua create mode 100644 test/sasl/suite.ini diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ccf792..c014d4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,6 +11,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH}) # Find Tarantool and Lua dependecies set(TARANTOOL_FIND_REQUIRED ON) find_package(Tarantool) +find_package(CyrusSASL) # include(cmake/FindTarantool.cmake) include_directories(${TARANTOOL_INCLUDE_DIRS}) diff --git a/README.md b/README.md index f757e81..e8b92c7 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ Memcached protocol 'wrapper' for tarantool. ### Prerequisites - * Tarantol 1.6.7+ with header files (tarantool && tarantool-dev packages). + * Tarantol 1.6.8+ with header files (tarantool && tarantool-dev packages). + * Cyrus SASL library (with header files) * Python >= 2.7, <3 with next packages (for testing only): - PyYAML - msgpack-python @@ -31,8 +32,9 @@ make make install ``` -Or use LuaRocks (in this case you'll need `libsmall`, `libsmall-dev` and `tarantool-dev` -packages available from our binary repository at http://tarantool.org/dist/master): +Or use LuaRocks (in this case you'll need `libsmall`, `libsmall-dev`, `tarantool-dev` +packages available from our binary repository at http://tarantool.org/dist/master, and +system package `libsasl2-dev`): ``` bash luarocks install https://raw.githubusercontent.com/tarantool/memcached/master/memcached-scm-1.rockspec --local @@ -104,6 +106,51 @@ END - ~~`disk` - store everything on hdd/ssd (using `sophia` engine)~~ (not yet supported) * *space_name* - custom name for a memcached space, default is `__mc_` * *if_not_exists* - do not throw error if an instance already exists. +* *sasl* - enable or disable SASL support (disabled by default) + +## SASL support + +Usual rules for memcached are aplicable for this plugin: + +1. Create user (NOTE: it'll use custom folder): + + ``` bash + echo testpass | saslpasswd2 -p -c testuser -f /tmp/test-tarantool-memcached.sasldb + ``` + +2. Place configuration file `/etc/sasl2/tarantool-memcached.conf`. For + + ``` + mech_list: plain cram-md5 + log_level: 7 + sasldb_path: /tmp/test-tarantool-memcached.sasldb + ``` + + NOTE: This will disable 'ANONYMOUS' (and other, that aren't listed) + authentication plugins. + NOTE: This will set logging level to the highest possible + NOTE: THis will set custom path for database path + +3. Run tarantool with memcached plugin with SASL enabled + + ``` + local memcached = require('memcached') + local instance = memcached.create('my_instance', '0.0.0.0:11211', { + sasl = true + }) + ``` + +4. Use your favorite binary(!!) memcached client, that supports(!!) SASL: + + Example using Python's ['python-binary-memcached' library](https://github.com/jaysonsantos/python-binary-memcached) + ``` + import bmemcached + client = bmemcached.Client(('127.0.0.1:11211', ), 'testuser', 'testpasswd') + client.set('key', 'value') + print client.get('key') + ``` + +For custom configuration file path, please, use `SASL_CONF_PATH` environment variable. ## What's supported, what's not and other features @@ -125,7 +172,7 @@ END - `append`/`prepend`/`incr`/`decr` - `verbosity` - partially, logging is not very good. - `stat` - `reset` is supported and all stats too. - - for now **SASL** authentication is not supported + - **SASL** authentication is supported - **range** operations are not supported as well. * Expiration is supported * Flush is supported diff --git a/cmake/FindCyrusSASL.cmake b/cmake/FindCyrusSASL.cmake new file mode 100644 index 0000000..8f92cf5 --- /dev/null +++ b/cmake/FindCyrusSASL.cmake @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# - Find Cyrus SASL (sasl.h, libsasl2.so) +# +# This module defines +# CYRUS_SASL_INCLUDE_DIR, directory containing headers +# CYRUS_SASL_SHARED_LIB, path to Cyrus SASL's shared library +# CYRUS_SASL_FOUND, whether Cyrus SASL and its plugins have been found +# +# N.B: we do _not_ include sasl in thirdparty, for a fairly subtle reason. The +# TLDR version is that newer versions of cyrus-sasl (>=2.1.26) have a bug fix +# for https://bugzilla.cyrusimap.org/show_bug.cgi?id=3590, but that bug fix +# relied on a change both on the plugin side and on the library side. If you +# then try to run the new version of sasl (e.g from our thirdparty tree) with +# an older version of a plugin (eg from RHEL6 install), you'll get a SASL_NOMECH +# error due to this bug. +# +# In practice, Cyrus-SASL is so commonly used and generally non-ABI-breaking that +# we should be OK to depend on the host installation. + + +find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h) +find_library(CYRUS_SASL_SHARED_LIB sasl2) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CYRUS_SASL REQUIRED_VARS + CYRUS_SASL_SHARED_LIB CYRUS_SASL_INCLUDE_DIR) diff --git a/debian/control b/debian/control index c8d3e07..44c9b28 100644 --- a/debian/control +++ b/debian/control @@ -5,7 +5,8 @@ Maintainer: Eugene Blikh Build-Depends: debhelper (>= 9), cdbs, cmake (>= 2.8), tarantool-dev (>= 1.6.8.0), - libsmall-dev, libmsgpuck-dev + libsmall-dev, libmsgpuck-dev, + libsasl2-dev Standards-Version: 3.9.6 Homepage: https://github.com/tarantool/memcached Vcs-Git: git://github.com/tarantool/memcached.git diff --git a/memcached/CMakeLists.txt b/memcached/CMakeLists.txt index 27c8078..9718250 100644 --- a/memcached/CMakeLists.txt +++ b/memcached/CMakeLists.txt @@ -17,6 +17,9 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E touch_nocreate # ${CMAKE_SOURCE_DIR}/memcached/internal/proto_txt_parser.c # PROPERTIES HEADER_FILE_ONLY true) +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-deprecated") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-deprecated") + add_library(internalso SHARED "internal/constants.c" "internal/utils.c" @@ -27,6 +30,7 @@ add_library(internalso SHARED "internal/memcached_layer.c" "internal/expiration.c" "internal/memcached.c" + "internal/mc_sasl.c" ) set_property(DIRECTORY PROPERTY CLEAN_NO_CUSTOM true) @@ -38,6 +42,7 @@ if(NOT SMALL_FOUND) endif() target_link_libraries(internalso small) +target_link_libraries(internalso sasl2) set_target_properties(internalso PROPERTIES diff --git a/memcached/init.lua b/memcached/init.lua index b7daf01..55b9817 100644 --- a/memcached/init.lua +++ b/memcached/init.lua @@ -70,18 +70,21 @@ enum memcached_options { MEMCACHED_OPT_FLUSH_ENABLED = 0x04, MEMCACHED_OPT_VERBOSITY = 0x05, MEMCACHED_OPT_PROTOCOL = 0x06, + MEMCACHED_OPT_SASL = 0x07, MEMCACHED_OPT_MAX }; -struct memcached_stat *memcached_get_stat (struct memcached_service *); +struct memcached_stat * +memcached_get_stat (struct memcached_service *); -struct memcached_service *memcached_create(const char *, uint32_t); -int memcached_start (struct memcached_service *); -void memcached_stop (struct memcached_service *); -void memcached_free (struct memcached_service *); +struct memcached_service * +memcached_create(const char *, uint32_t); -void -memcached_handler(struct memcached_service *p, int fd); +int memcached_start (struct memcached_service *); +void memcached_stop (struct memcached_service *); +void memcached_free (struct memcached_service *); + +void memcached_handler(struct memcached_service *p, int fd); ]] function startswith(str, start) @@ -156,6 +159,12 @@ local typetable = { end, [[protocol type ('negotiation'/'binary'/'ascii')]] }, + sasl = { + 'boolean', + function() return false end, + function(x) return x end, + [[enable SASL authorization]] + }, storage = { 'string', function() return 'memory' end, @@ -239,10 +248,11 @@ local conf_table = { expire_items_per_iter = ffi.C.MEMCACHED_OPT_EXPIRE_COUNT, expire_full_scan_time = ffi.C.MEMCACHED_OPT_EXPIRE_TIME, verbosity = ffi.C.MEMCACHED_OPT_VERBOSITY, - protocol = ffi.C.MEMCACHED_OPT_PROTOCOL + protocol = ffi.C.MEMCACHED_OPT_PROTOCOL, + sasl = ffi.C.MEMCACHED_OPT_SASL } -local memcached_mt = { +local memcached_methods = { cfg = function (self, opts) if type(opts) ~= 'table' then error(err_bad_args) @@ -270,10 +280,8 @@ local memcached_mt = { end ffi.C.memcached_start(self.service) local parsed = uri.parse(self.uri) - self.listener = socket.tcp_server( - parsed.host, - parsed.service, { - handler = memcached_handler + self.listener = socket.tcp_server(parsed.host, parsed.service, { + handler = memcached_handler }) if self.listener == nil then self.status = ERRORED @@ -291,6 +299,7 @@ local memcached_mt = { end local rc = ffi.C.memcached_stop(self.service) self.status = STOPPED + return self end, info = function (self) stats = ffi.C.memcached_get_stat(self.service) @@ -299,6 +308,10 @@ local memcached_mt = { retval[v] = stats[0][v] end return retval + end, + grant = function (self, username) + box.schema.user.grant(username, 'read,write', 'space', self.space_name) + return self end } @@ -322,12 +335,12 @@ local function memcached_init(name, uri, opts) instance.space = box.schema.create_space(instance.space_name, { engine = storage, format = { - { name = 'key', type = 'str' }, - { name = 'expire', type = 'num' }, + { name = 'key', type = 'str' }, + { name = 'expire', type = 'num' }, { name = 'creation', type = 'num' }, - { name = 'value', type = 'str' }, - { name = 'cas', type = 'num' }, - { name = 'flags', type = 'num' }, + { name = 'value', type = 'str' }, + { name = 'cas', type = 'num' }, + { name = 'flags', type = 'num' }, } }) instance.space:create_index('primary', { @@ -342,14 +355,20 @@ local function memcached_init(name, uri, opts) error(fmt(err_enomem, "memcached service")) end instance.service = ffi.gc(service, ffi.C.memcached_free) - memcached_services[instance.name] = setmetatable(instance, - { __index = memcached_mt } - ) + memcached_services[instance.name] = setmetatable(instance, { + __index = memcached_methods + }) return instance:cfg(opts):start() end +local function memcached_get(name) + return memcached_services[name] +end + return { - create = memcached_init; - get = function (name) return memcached_services[name] end; - debug = memcached_services; + create = memcached_init, + get = memcached_get, + server = setmetatable({}, { + __index = memcached_services + }) } diff --git a/memcached/internal/constants.h b/memcached/internal/constants.h index f7134be..0af4704 100644 --- a/memcached/internal/constants.h +++ b/memcached/internal/constants.h @@ -252,6 +252,12 @@ struct memcached_response_record { /* Not presented in vanilla memcached */ \ /* 0x86 */ _(MEMCACHED_RES_EAGAIN, 2, "Temporary unavailable") +enum memcached_authentication_state { + MEMCACHED_AUTH_NOT = 0x00, + MEMCACHED_AUTH_STEP = 0x01, + MEMCACHED_AUTH_OK = 0x02 +}; + ENUM0(memcached_response, RESPONSE_CODES); ENUM0_TXT(memcached_txt_cmd, TEXT_COMMANDS); diff --git a/memcached/internal/expiration.c b/memcached/internal/expiration.c index f1436b0..5b607ee 100644 --- a/memcached/internal/expiration.c +++ b/memcached/internal/expiration.c @@ -24,7 +24,9 @@ memcached_expire_process(struct memcached_service *p, box_iterator_t **iterp) return -1; } else if (tpl == NULL) { box_iterator_free(iter); - box_txn_commit(); + if (box_txn_commit() == -1) { + return -1; + } *iterp = NULL; return 0; } else if (is_expired_tuple(p, tpl)) { @@ -47,7 +49,9 @@ memcached_expire_process(struct memcached_service *p, box_iterator_t **iterp) p->stat.evictions++; } } - box_txn_commit(); + if (box_txn_commit() == -1) { + return -1; + } return 0; } @@ -71,6 +75,13 @@ memcached_expire_loop(va_list ap) goto finish; } rv = memcached_expire_process(p, &iter); + if (rv == -1) { + const box_error_t *err = box_error_last(); + say_error("Unexpected error %u: %s", + box_error_code(err), + box_error_message(err)); + goto finish; + } /* This part is where we rest after all deletes */ double delay = ((double )p->expire_count * p->expire_time) / @@ -84,7 +95,8 @@ memcached_expire_loop(va_list ap) goto restart; finish: - if (iter) box_iterator_free(iter); + if (iter) + box_iterator_free(iter); return 0; } diff --git a/memcached/internal/mc_sasl.c b/memcached/internal/mc_sasl.c new file mode 100644 index 0000000..594b76a --- /dev/null +++ b/memcached/internal/mc_sasl.c @@ -0,0 +1,270 @@ +#include +#include +#include +#include + +#include "memcached.h" +#include "mc_sasl.h" + +/******************************************************************************/ +/* wrappers on cyrus sasl lib */ +/******************************************************************************/ + +#include +#include + +#include + +const char mc_auth_ok_response[] = "Authenticated"; +size_t mc_auth_ok_response_len = sizeof(mc_auth_ok_response) - 1; + +char my_sasl_hostname[1025]; + +/******************************************************************************/ +/* password database related functions */ +/******************************************************************************/ + +#if 0 +static const char *memcached_sasl_pwdb_c = "MEMCACHED_SASL_PWDB"; + +#define MAX_ENTRY_LEN 256 + +static const char *memcached_sasl_pwdb; + +static int sasl_check_token(char tkn) { + return (tkn == ':' || tkn == '\n' || tkn == '\r' || tkn == '\0'); +} + +static int memcached_sasl_server_userdb_checkpass( + sasl_conn_t *conn, void *context, const char *user, + const char *pwd, unsigned pwd_len, struct propctx *propctx) { + (void )conn; + (void )context; + (void )propctx; + size_t user_len = strlen(user); + if ((pwd_len + user_len) > (MAX_ENTRY_LEN - 4)) { + say_error("<%s>: Too long request", __func__); + return SASL_NOAUTHZ; + } + + FILE *pwd_f = fopen(memcached_sasl_pwdb, "r"); + if (pwd_f == NULL) { + say_error("<%s>: Failed to open database", __func__); + return SASL_NOAUTHZ; + } + + char buffer[MAX_ENTRY_LEN]; + bool ok = false; + + while (fgets(buffer, sizeof(buffer), pwd_f) != NULL) { + if (memcmp(user, buffer, user_len) == 0 && + buffer[user_len] == ':') { + ++user_len; + if (memcmp(pwd, buffer + user_len, pwd_len) == 0 && + sasl_check_token(buffer[user_len + pwd_len])) { + ok = true; + } + break; + } + } + if (ok == false) { + say_error("<%s>: Failed to authenticate", __func__); + } + + fclose(pwd_f); + + return (ok ? SASL_OK : SASL_NOAUTHZ); +} +#endif + +#ifdef SASL_CB_GETCONF +static const char *memcached_sasl_conf_path_c = "SASL_CONF_PATH"; + +static const char *const memcached_sasl_getconf_locations[] = { + "/etc/sasl/tarantool-memcached.conf", + "/etc/sasl2/tarantool-memcached.conf", + NULL +}; + +__attribute__((unused)) +static int memcached_sasl_getconf(void *context, const char **path) { + (void )context; + *path = getenv(memcached_sasl_conf_path_c); + + if (*path == NULL) { + for (int i = 0; memcached_sasl_getconf_locations[i] != NULL; ++i) { + if (access(memcached_sasl_getconf_locations[i], F_OK) == 0) { + *path = memcached_sasl_getconf_locations[i]; + break; + } + } + } + + if (*path != NULL) { + say_info("Found configuration file for SASL in '%s'", *path); + } else { + say_error("Can't find configuration file for SASL"); + } + + return (*path != NULL ? SASL_OK : SASL_FAIL); +} +#endif + +static int memcached_sasl_log(void *context, int level, const char *message) { + (void )context; + int tnt_level = S_INFO; + + switch (level) { + case SASL_LOG_PASS: + case SASL_LOG_TRACE: + case SASL_LOG_NOTE: + tnt_level = S_INFO; + break; + case SASL_LOG_DEBUG: + case SASL_LOG_NONE: + tnt_level = S_DEBUG; + break; + case SASL_LOG_WARN: + tnt_level = S_WARN; + break; + case SASL_LOG_FAIL: + tnt_level = S_ERROR; + break; + default: + /* unreacheable, becomes fatal */ + break; + } + + say(tnt_level, NULL, "SASL %s", level, message); + + return SASL_OK; +} + +static sasl_callback_t sasl_callbacks[] = { + { SASL_CB_LOG, (sasl_callback_ft )&memcached_sasl_log, NULL}, +#ifdef SASL_CB_GETCONF + { SASL_CB_GETCONF, (sasl_callback_ft )&memcached_sasl_getconf, NULL}, +#endif +#if 0 + { + SASL_CB_SERVER_USERDB_CHECKPASS, + (sasl_callback_ft )&memcached_sasl_server_userdb_checkpass, + NULL + }, +#endif + { SASL_CB_LIST_END, NULL, NULL}, +}; + +/* PUBLIC API */ + +int memcached_sasl_init(void) { + say_info("Initializing SASL: begin"); + +#if 0 + memcached_sasl_pwdb = getenv(memcached_sasl_pwdb_c); + if (memcached_sasl_pwdb == NULL) { + say_warn("PWDB isn't specified. skipping"); + sasl_callbacks[0].id = SASL_CB_LIST_END; + sasl_callbacks[0].proc = NULL; + } +#endif + + memset(my_sasl_hostname, 0, sizeof(my_sasl_hostname)); + if (gethostname(my_sasl_hostname, sizeof(my_sasl_hostname) - 1) == -1) { + say_syserror("Initializing SASL: Failed to discover hostname"); + my_sasl_hostname[0] = '\0'; + } + + if (sasl_server_init(sasl_callbacks, "tarantool-memcached") != SASL_OK){ + say_error("Initializing SASL: Failed"); + return -1; + } + + say_info("Initializing SASL: done"); + return 0; +} + +int memcached_sasl_connection_init(struct memcached_connection *con) { + const char *hostname = NULL; + if (my_sasl_hostname[0] != '\0') { + hostname = (const char *)my_sasl_hostname; + } + struct sasl_ctx *ctx = con->sasl_ctx; + int result = sasl_server_new("tarantool-memcached", NULL, hostname, + NULL, NULL, NULL, 0, &ctx->sasl_conn); + if (result != SASL_OK) { + say_error("Failed to initialize SASL"); + ctx->sasl_conn = NULL; + return -1; + } + return 0; +} + +int memcached_sasl_list_mechs(struct memcached_connection *con, const char **mechs, + size_t *mechs_len) { + unsigned int mechs_len_ui = 0; + struct sasl_ctx *ctx = con->sasl_ctx; + int result = sasl_listmech(ctx->sasl_conn, NULL, "", " ", "", mechs, + &mechs_len_ui, NULL); + if (result == SASL_OK) { + *mechs_len = mechs_len_ui; + return 0; + } + say_error("<%s>: Failed with exit code %d", __func__, + result); + *mechs = NULL; + return -1; +} + +/** + * 0 means OK - authentication done + * 1 means STEP - next step of authentication + * -1 means ERROR + */ +int memcached_sasl_auth(struct memcached_connection *con, const char *mech, + const char *challenge, size_t challenge_len, + const char **out, size_t *out_len) { + struct sasl_ctx *ctx = con->sasl_ctx; + unsigned int challenge_len_ui = (unsigned int )challenge_len; + unsigned int out_len_ui = 0; + int result = sasl_server_start(ctx->sasl_conn, mech, challenge, + challenge_len_ui, out, &out_len_ui); + *out_len = (size_t )out_len_ui; + if (result == SASL_OK) { + *out = mc_auth_ok_response; + *out_len = mc_auth_ok_response_len; + return 0; + } else if (result == SASL_CONTINUE) { + return 1; + } + say_error("<%s>: Failed with exit code %d", __func__, + result); + return -1; +} + +/** + * 0 means OK - authentication done + * 1 means STEP - next step of authentication + * -1 means ERROR + */ +int memcached_sasl_step(struct memcached_connection *con, + const char *challenge, size_t challenge_len, + const char **out, size_t *out_len) { + struct sasl_ctx *ctx = con->sasl_ctx; + unsigned int challenge_len_ui = (unsigned int )challenge_len; + unsigned int out_len_ui = 0; + int result = sasl_server_step(ctx->sasl_conn, challenge, + challenge_len_ui, out, &out_len_ui); + *out_len = (size_t )out_len_ui; + if (result == SASL_OK) { + *out = mc_auth_ok_response; + *out_len = mc_auth_ok_response_len; + return 0; + } else if (result == SASL_CONTINUE) { + return 1; + } + say_error("<%s>: Failed with exit code %d", __func__, + result); + return -1; +} + diff --git a/memcached/internal/mc_sasl.h b/memcached/internal/mc_sasl.h new file mode 100644 index 0000000..567b55a --- /dev/null +++ b/memcached/internal/mc_sasl.h @@ -0,0 +1,28 @@ +#ifndef MC_SASL_H_INCLUDED +#define MC_SASL_H_INCLUDED + +struct sasl_conn; +typedef struct sasl_conn sasl_conn_t; + +/* + * struct sasl_conn; + * typedef struct sasl_conn sasl_conn_t; + * typedef int (sasl_callback_ft )(void); + */ + +struct sasl_ctx { + sasl_conn_t *sasl_conn; +}; + +int memcached_sasl_init(void); +int memcached_sasl_connection_init(struct memcached_connection *con); +int memcached_sasl_list_mechs(struct memcached_connection *con, + const char **mechs, size_t *mechs_len); +int memcached_sasl_auth(struct memcached_connection *con, const char *mech, + const char *challenge, size_t challenge_len, + const char **out, size_t *out_len); +int memcached_sasl_step(struct memcached_connection *con, + const char *challenge, size_t challenge_len, + const char **out, size_t *out_len); + +#endif /* MC_SASL_H_INCLUDED */ diff --git a/memcached/internal/memcached.c b/memcached/internal/memcached.c index 6d36576..f2d2400 100644 --- a/memcached/internal/memcached.c +++ b/memcached/internal/memcached.c @@ -44,6 +44,7 @@ #include "proto_bin.h" #include "proto_txt.h" #include "expiration.h" +#include "mc_sasl.h" static inline int memcached_skip_request(struct memcached_connection *con) { @@ -203,16 +204,22 @@ memcached_handler(struct memcached_service *p, int fd) struct memcached_connection con; /* TODO: move to connection_init */ memset(&con, 0, sizeof(struct memcached_connection)); - con.fd = fd; - con.in = ibuf_new(); - con.out = obuf_new(); - con.write_end = obuf_create_svp(con.out); - con.cfg = p; + con.fd = fd; + con.in = ibuf_new(); + con.out = obuf_new(); + con.write_end = obuf_create_svp(con.out); + con.cfg = p; + con.authentication_step = false; + con.sasl_ctx = (struct sasl_ctx *)calloc(1, + sizeof(struct sasl_ctx)); + if (memcached_sasl_connection_init(&con)) { + return; + } /* prepare connection type */ - if (p->proto == MEMCACHED_PROTO_NEGOTIATION) { + if (p->proto == MEMCACHED_PROTO_NEGOTIATION && p->sasl != true) { con.cb.parse_request = memcached_loop_negotiate; - } else if (p->proto == MEMCACHED_PROTO_BINARY) { + } else if (p->proto == MEMCACHED_PROTO_BINARY || p->sasl == true) { memcached_set_bin(&con); } else if (p->proto == MEMCACHED_PROTO_TEXT) { memcached_set_txt(&con); @@ -237,6 +244,7 @@ struct memcached_service* memcached_create(const char *name, uint32_t sid) { iobuf_mempool_create(); + memcached_sasl_init(); struct memcached_service *srv = (struct memcached_service *) calloc(1, sizeof(struct memcached_service)); if (!srv) { @@ -332,11 +340,30 @@ memcached_set_opt (struct memcached_service *srv, int opt, ...) srv->proto = MEMCACHED_PROTO_BINARY; } else if (strncmp(type, "negot", 5) == 0) { srv->proto = MEMCACHED_PROTO_NEGOTIATION; - } else { + if (srv->sasl == true) + srv->proto = MEMCACHED_PROTO_BINARY; + } else if (strncmp(type, "text", 4) == 0) { + if (srv->sasl == true) { + say_error("Can't set Text protocol. SASL authen" + "tication is enabled."); + return; + } srv->proto = MEMCACHED_PROTO_TEXT; + } else { + /* unreacheable */ + assert(0); } break; } + case MEMCACHED_OPT_SASL: + if (srv->proto == MEMCACHED_PROTO_TEXT) { + say_error("Can't enable SASL authentication. Text proto" + "col is enabled."); + srv->sasl = false; + break; + } + srv->sasl = true; + break; default: say_error("No such option %d", opt); break; diff --git a/memcached/internal/memcached.h b/memcached/internal/memcached.h index 333502c..7b41c48 100644 --- a/memcached/internal/memcached.h +++ b/memcached/internal/memcached.h @@ -81,6 +81,7 @@ struct memcached_service { const char *uri; const char *name; uint32_t space_id; + bool sasl; /* properties */ uint64_t cas; uint64_t flush; @@ -121,6 +122,8 @@ struct memcached_connection { // }; // socklen_t addr_len; // struct session *session; + struct sasl_ctx *sasl_ctx; + int authentication_step; union { /* request data (binary) */ struct { @@ -147,7 +150,7 @@ memcached_get_stat(struct memcached_service *); struct memcached_service * memcached_create(const char *, uint32_t); -int memcached_start(struct memcached_service *); +int memcached_start(struct memcached_service *); void memcached_stop(struct memcached_service *); void memcached_free(struct memcached_service *); @@ -163,6 +166,7 @@ enum memcached_options { MEMCACHED_OPT_FLUSH_ENABLED = 0x04, MEMCACHED_OPT_VERBOSITY = 0x05, MEMCACHED_OPT_PROTOCOL = 0x06, + MEMCACHED_OPT_SASL = 0x07, MEMCACHED_OPT_MAX }; diff --git a/memcached/internal/proto_bin.c b/memcached/internal/proto_bin.c index 6cb5a25..303725f 100644 --- a/memcached/internal/proto_bin.c +++ b/memcached/internal/proto_bin.c @@ -6,13 +6,14 @@ #include #include +#include "memcached.h" +#include "proto_bin.h" + #include "error.h" #include "utils.h" -#include "memcached.h" #include "constants.h" #include "memcached_layer.h" - -#include "proto_bin.h" +#include "mc_sasl.h" #include #include @@ -78,6 +79,24 @@ write_output_ok_cas(struct memcached_connection *con, uint64_t cas) return write_output_ok(con, cas, 0, 0, 0, NULL, NULL, NULL); } +static inline bool +is_authenticated(struct memcached_connection *con) +{ + /* default declarations */ + struct memcached_hdr *h = con->hdr; + bool rv = (con->authentication_step == MEMCACHED_AUTH_OK); + + if (con->cfg->sasl == false || + h->cmd == MEMCACHED_BIN_CMD_SASL_LIST_MECHS || + h->cmd == MEMCACHED_BIN_CMD_SASL_AUTH || + h->cmd == MEMCACHED_BIN_CMD_SASL_STEP || + h->cmd == MEMCACHED_BIN_CMD_VERSION) { + rv = true; + } + + return rv; +} + /** * ext/key/val - -1 - must not be presented * 0 - may be presented @@ -790,68 +809,157 @@ memcached_bin_process_stat(struct memcached_connection *con) { return 0; } +int +memcached_bin_process_sasl_list_mech(struct memcached_connection *con) { + + /* default declarations */ + if (con->cfg->sasl == false) { + return memcached_process_unknown(con); + } + + const char *mechs = NULL; + size_t mechs_len = 0; + + int rv = memcached_sasl_list_mechs(con, &mechs, &mechs_len); + + if (rv < 0) { + memcached_error(MEMCACHED_RES_AUTH_ERROR); + return -1; + } + + memcached_bin_write(con, MEMCACHED_RES_OK, 0, + 0, 0, mechs_len, + NULL, NULL, mechs); + return 0; +} + +int +memcached_bin_process_sasl_auth(struct memcached_connection *con) { + + /* default declarations */ + struct memcached_hdr *h = con->hdr; + struct memcached_body *b = &con->body; + + if (con->cfg->sasl == false) { + return memcached_process_unknown(con); + } + + int rv = 0; + + char mech[257] = {'\0'}; + memcpy(mech, b->key, b->key_len); + mech[b->key_len] = '\0'; + + const char *challenge = b->val; + size_t challenge_len = b->val_len; + + const char *out = NULL; + size_t out_len = 0; + + switch (h->cmd) { + case MEMCACHED_BIN_CMD_SASL_AUTH: + rv = memcached_sasl_auth(con, mech, challenge, challenge_len, + &out, &out_len); + con->authentication_step = MEMCACHED_AUTH_STEP; + break; + case MEMCACHED_BIN_CMD_SASL_STEP: + if (con->authentication_step != MEMCACHED_AUTH_STEP) { + memcached_error(MEMCACHED_RES_AUTH_ERROR); + return -1; + } + rv = memcached_sasl_step(con, challenge, challenge_len, + &out, &out_len); + break; + default: + /* unreacheable */ + assert(0); + } + + con->cfg->stat.auth_cmds += 1; + switch (rv) { + case -1: + con->cfg->stat.auth_errors += 1; + memcached_error(MEMCACHED_RES_AUTH_ERROR); + /* con->close_connection = true; */ + return -1; + case 0: + con->authentication_step = MEMCACHED_AUTH_OK; + /* FALLTHROUGH */ + case 1: + memcached_bin_write(con, MEMCACHED_RES_OK, 0, + 0, 0, out_len, + NULL, NULL, out); + break; + default: + /* unreachable */ + assert(0); + } + + return 0; +} + const mc_process_func_t memcached_bin_handler[] = { - memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GET , 0x00 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_SET , 0x01 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_ADD , 0x02 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_REPLACE , 0x03 */ - memcached_bin_process_delete, /* MEMCACHED_BIN_CMD_DELETE , 0x04 */ - memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_INCR , 0x05 */ - memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_DECR , 0x06 */ - memcached_bin_process_quit, /* MEMCACHED_BIN_CMD_QUIT , 0x07 */ - memcached_bin_process_flush, /* MEMCACHED_BIN_CMD_FLUSH , 0x08 */ - memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETQ , 0x09 */ - memcached_bin_process_noop, /* MEMCACHED_BIN_CMD_NOOP , 0x0a */ - memcached_bin_process_version, /* MEMCACHED_BIN_CMD_VERSION , 0x0b */ - memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETK , 0x0c */ - memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETKQ , 0x0d */ - memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_APPEND , 0x0e */ - memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_PREPEND , 0x0f */ - memcached_bin_process_stat, /* MEMCACHED_BIN_CMD_STAT , 0x10 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_SETQ , 0x11 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_ADDQ , 0x12 */ - memcached_bin_process_set, /* MEMCACHED_BIN_CMD_REPLACEQ , 0x13 */ - memcached_bin_process_delete, /* MEMCACHED_BIN_CMD_DELETEQ , 0x14 */ - memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_INCRQ , 0x15 */ - memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_DECRQ , 0x16 */ - memcached_bin_process_quit, /* MEMCACHED_BIN_CMD_QUITQ , 0x17 */ - memcached_bin_process_flush, /* MEMCACHED_BIN_CMD_FLUSHQ , 0x18 */ - memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_APPENDQ , 0x19 */ - memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_PREPENDQ , 0x1a */ - memcached_bin_process_verbosity, /* MEMCACHED_BIN_CMD_VERBOSITY, 0x1b */ - memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_TOUCH , 0x1c */ - memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GAT , 0x1d */ - memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATQ , 0x1e */ - memcached_process_unknown, /* RESERVED , 0x1f */ - memcached_process_unsupported, /* MEMCACHED_._SASL_LIST_MECHS, 0x20 */ - memcached_process_unsupported, /* MEMCACHED_._SASL_AUTH , 0x21 */ - memcached_process_unsupported, /* MEMCACHED_._SASL_STEP , 0x22 */ - memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATK , 0x23 */ - memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATKQ , 0x24 */ - memcached_process_unknown, /* RESERVED , 0x25 */ - memcached_process_unknown, /* RESERVED , 0x26 */ - memcached_process_unknown, /* RESERVED , 0x27 */ - memcached_process_unknown, /* RESERVED , 0x28 */ - memcached_process_unknown, /* RESERVED , 0x29 */ - memcached_process_unknown, /* RESERVED , 0x2a */ - memcached_process_unknown, /* RESERVED , 0x2b */ - memcached_process_unknown, /* RESERVED , 0x2c */ - memcached_process_unknown, /* RESERVED , 0x2d */ - memcached_process_unknown, /* RESERVED , 0x2e */ - memcached_process_unknown, /* RESERVED , 0x2f */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RGET , 0x30 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RSET , 0x31 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RSETQ , 0x32 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RAPPEND , 0x33 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RAPPENDQ , 0x34 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RPREPEND , 0x35 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RPREPENDQ, 0x36 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDELETE , 0x37 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDELETEQ , 0x38 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RINCR , 0x39 */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RINCRQ , 0x3a */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDECR , 0x3b */ - memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDECRQ , 0x3c */ + memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GET , 0x00 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_SET , 0x01 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_ADD , 0x02 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_REPLACE , 0x03 */ + memcached_bin_process_delete, /* MEMCACHED_BIN_CMD_DELETE , 0x04 */ + memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_INCR , 0x05 */ + memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_DECR , 0x06 */ + memcached_bin_process_quit, /* MEMCACHED_BIN_CMD_QUIT , 0x07 */ + memcached_bin_process_flush, /* MEMCACHED_BIN_CMD_FLUSH , 0x08 */ + memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETQ , 0x09 */ + memcached_bin_process_noop, /* MEMCACHED_BIN_CMD_NOOP , 0x0a */ + memcached_bin_process_version, /* MEMCACHED_BIN_CMD_VERSION , 0x0b */ + memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETK , 0x0c */ + memcached_bin_process_get, /* MEMCACHED_BIN_CMD_GETKQ , 0x0d */ + memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_APPEND , 0x0e */ + memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_PREPEND , 0x0f */ + memcached_bin_process_stat, /* MEMCACHED_BIN_CMD_STAT , 0x10 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_SETQ , 0x11 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_ADDQ , 0x12 */ + memcached_bin_process_set, /* MEMCACHED_BIN_CMD_REPLACEQ , 0x13 */ + memcached_bin_process_delete, /* MEMCACHED_BIN_CMD_DELETEQ , 0x14 */ + memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_INCRQ , 0x15 */ + memcached_bin_process_delta, /* MEMCACHED_BIN_CMD_DECRQ , 0x16 */ + memcached_bin_process_quit, /* MEMCACHED_BIN_CMD_QUITQ , 0x17 */ + memcached_bin_process_flush, /* MEMCACHED_BIN_CMD_FLUSHQ , 0x18 */ + memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_APPENDQ , 0x19 */ + memcached_bin_process_pend, /* MEMCACHED_BIN_CMD_PREPENDQ , 0x1a */ + memcached_bin_process_verbosity, /* MEMCACHED_BIN_CMD_VERBOSITY, 0x1b */ + memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_TOUCH , 0x1c */ + memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GAT , 0x1d */ + memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATQ , 0x1e */ + memcached_process_unknown, /* RESERVED , 0x1f */ + memcached_bin_process_sasl_list_mech, /* MEMCACHED_._SASL_LIST_MECHS, 0x20 */ + memcached_bin_process_sasl_auth, /* MEMCACHED_._SASL_AUTH , 0x21 */ + memcached_bin_process_sasl_auth, /* MEMCACHED_._SASL_STEP , 0x22 */ + memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATK , 0x23 */ + memcached_bin_process_gat, /* MEMCACHED_BIN_CMD_GATKQ , 0x24 */ + memcached_process_unknown, /* RESERVED , 0x25 */ + memcached_process_unknown, /* RESERVED , 0x26 */ + memcached_process_unknown, /* RESERVED , 0x27 */ + memcached_process_unknown, /* RESERVED , 0x28 */ + memcached_process_unknown, /* RESERVED , 0x29 */ + memcached_process_unknown, /* RESERVED , 0x2a */ + memcached_process_unknown, /* RESERVED , 0x2b */ + memcached_process_unknown, /* RESERVED , 0x2c */ + memcached_process_unknown, /* RESERVED , 0x2d */ + memcached_process_unknown, /* RESERVED , 0x2e */ + memcached_process_unknown, /* RESERVED , 0x2f */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RGET , 0x30 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RSET , 0x31 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RSETQ , 0x32 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RAPPEND , 0x33 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RAPPENDQ , 0x34 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RPREPEND , 0x35 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RPREPENDQ, 0x36 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDELETE , 0x37 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDELETEQ , 0x38 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RINCR , 0x39 */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RINCRQ , 0x3a */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDECR , 0x3b */ + memcached_process_unsupported, /* MEMCACHED_BIN_CMD_RDECRQ , 0x3c */ NULL }; @@ -884,12 +992,19 @@ memcached_bin_process(struct memcached_connection *con) int rv = 0; /* Process message */ con->noreply = false; + if (is_authenticated(con) == false) { + memcached_error(MEMCACHED_RES_AUTH_ERROR); + /* TODO: really need this? */ + /* con->close_connection = true; */ + return -1; + } if (memcached_bin_ntxn(con)) { box_txn_begin(); } if (con->hdr->cmd < memcached_bin_cmd_MAX) { rv = memcached_bin_handler[con->hdr->cmd](con); - if (box_txn()) box_txn_commit(); + if (box_txn()) + box_txn_commit(); } else { rv = memcached_process_unknown(con); } @@ -969,10 +1084,6 @@ memcached_bin_parse(struct memcached_connection *con) } /** - * Write error code message to output buffer - * - * \param[in] con Connection object for writing output to - * \param[in] err Error code (status) * \param[in] errstr Eror string (if available) with description (may be NULL) * */ diff --git a/memcached/internal/proto_txt.c b/memcached/internal/proto_txt.c index 1021c0e..c227972 100644 --- a/memcached/internal/proto_txt.c +++ b/memcached/internal/proto_txt.c @@ -138,6 +138,8 @@ memcached_txt_process_set(struct memcached_connection *con) bool tuple_exists = (tuple != NULL); bool tuple_expired = tuple_exists && is_expired_tuple(con->cfg, tuple); + con->cfg->stat.cmd_set++; + /* Check for key (non)existence for different commands */ if (cmd == MEMCACHED_TXT_CMD_REPLACE && (!tuple_exists || tuple_expired)) { memcached_txt_DUP(con, "NOT_STORED\r\n", 12); diff --git a/rpm/tarantool-memcached.spec b/rpm/tarantool-memcached.spec index 2d262d7..45ce5bf 100644 --- a/rpm/tarantool-memcached.spec +++ b/rpm/tarantool-memcached.spec @@ -12,6 +12,7 @@ BuildRequires: tarantool-devel >= 1.6.8.0 BuildRequires: small-devel BuildRequires: msgpuck-devel BuildRequires: /usr/bin/prove +BuildRequires: cyrus-sasl-devel Requires: tarantool >= 1.6.8.0 %description diff --git a/test-run b/test-run index a57efaf..3b4942b 160000 --- a/test-run +++ b/test-run @@ -1 +1 @@ -Subproject commit a57efafb75520e2fffdc43af796655d99c3cb1dd +Subproject commit 3b4942b23e0a6b042747415e7d79bb91213e2acf diff --git a/test.sh b/test.sh index 2aae2d8..880e93c 100644 --- a/test.sh +++ b/test.sh @@ -1,5 +1,5 @@ curl -s https://packagecloud.io/install/repositories/tarantool/1_6/script.deb.sh | sudo bash -sudo apt-get install -y tarantool tarantool-dev --force-yes +sudo apt-get install -y tarantool tarantool-dev libsasl2-dev sasl2-bin --force-yes pip install --user python-daemon PyYAML six==1.9.0 TARANTOOL_DIR=/usr/include cmake . -DCMAKE_BUILD_TYPE=Release make internalso libmemcached diff --git a/test/.tarantoolctl b/test/.tarantoolctl index 94bb001..b6e373b 100644 --- a/test/.tarantoolctl +++ b/test/.tarantoolctl @@ -1,15 +1,21 @@ -- Options for test-run tarantoolctl local workdir = os.getenv('TEST_WORKDIR') + default_cfg = { pid_file = workdir, wal_dir = workdir, snap_dir = workdir, - sophia_dir = workdir, logger = workdir, background = false, } +if _TARANTOOL:match("1.6") ~= nil then + default_cfg.sophia_dir = workdir +else + default_cfg.vinyl_dir = workdir +end + instance_dir = workdir -- vim: set ft=lua : diff --git a/test/internal/memcached_connection.py b/test/internal/memcached_connection.py index bd20946..aa8a57a 100644 --- a/test/internal/memcached_connection.py +++ b/test/internal/memcached_connection.py @@ -20,39 +20,44 @@ 'response': 0x81 } +# Operation: ID, header type, extension length, key length, val length +# key/val length: 0 means nothing must be. 'None' means should be presented COMMANDS = { - 'get' : [0x00, Struct(HEADER + ''), 0, None, 0], - 'set' : [0x01, Struct(HEADER + 'LL'), 8, None, None], - 'add' : [0x02, Struct(HEADER + 'LL'), 8, None, None], - 'replace' : [0x03, Struct(HEADER + 'LL'), 8, None, None], - 'delete' : [0x04, Struct(HEADER + ''), 0, None, 0], - 'incr' : [0x05, Struct(HEADER + 'QQL'), 20, None, 0], - 'decr' : [0x06, Struct(HEADER + 'QQL'), 20, None, 0], - 'quit' : [0x07, Struct(HEADER + ''), 0, 0, 0], - 'flush' : [0x08, Struct(HEADER + 'I'), 4, 0, 0], - 'getq' : [0x09, Struct(HEADER + ''), 0, None, 0], - 'noop' : [0x0a, Struct(HEADER + ''), 0, 0, 0], - 'version' : [0x0b, Struct(HEADER + ''), 0, 0, 0], - 'getk' : [0x0c, Struct(HEADER + ''), 0, None, 0], - 'getkq' : [0x0d, Struct(HEADER + ''), 0, None, 0], - 'append' : [0x0e, Struct(HEADER + ''), 0, None, None], - 'prepend' : [0x0f, Struct(HEADER + ''), 0, None, None], - 'stat' : [0x10, Struct(HEADER + ''), 0, None, 0], - 'setq' : [0x11, Struct(HEADER + 'LL'), 8, None, None], - 'addq' : [0x12, Struct(HEADER + 'LL'), 8, None, None], - 'replaceq': [0x13, Struct(HEADER + 'LL'), 8, None, None], - 'deleteq' : [0x14, Struct(HEADER + ''), 0, None, 0], - 'incrq' : [0x15, Struct(HEADER + 'QQL'), 20, None, 0], - 'decrq' : [0x16, Struct(HEADER + 'QQL'), 20, None, 0], - 'quitq' : [0x17, Struct(HEADER + ''), 0, 0, 0], - 'flushq' : [0x18, Struct(HEADER + 'I'), 4, 0, 0], - 'appendq' : [0x19, Struct(HEADER + ''), 0, None, None], - 'prependq': [0x1A, Struct(HEADER + ''), 0, None, None], - 'touch' : [0x1C, Struct(HEADER + 'L'), 4, None, 0], - 'gat' : [0x1D, Struct(HEADER + 'L'), 4, None, 0], - 'gatq' : [0x1E, Struct(HEADER + 'L'), 4, None, 0], - 'gatk' : [0x23, Struct(HEADER + 'L'), 4, None, 0], - 'gatkq' : [0x24, Struct(HEADER + 'L'), 4, None, 0], + 'get' : [0x00, Struct(HEADER + ''), 0, None, 0 ], + 'set' : [0x01, Struct(HEADER + 'LL'), 8, None, None], + 'add' : [0x02, Struct(HEADER + 'LL'), 8, None, None], + 'replace' : [0x03, Struct(HEADER + 'LL'), 8, None, None], + 'delete' : [0x04, Struct(HEADER + ''), 0, None, 0 ], + 'incr' : [0x05, Struct(HEADER + 'QQL'), 20, None, 0 ], + 'decr' : [0x06, Struct(HEADER + 'QQL'), 20, None, 0 ], + 'quit' : [0x07, Struct(HEADER + ''), 0, 0, 0 ], + 'flush' : [0x08, Struct(HEADER + 'I'), 4, 0, 0 ], + 'getq' : [0x09, Struct(HEADER + ''), 0, None, 0 ], + 'noop' : [0x0a, Struct(HEADER + ''), 0, 0, 0 ], + 'version' : [0x0b, Struct(HEADER + ''), 0, 0, 0 ], + 'getk' : [0x0c, Struct(HEADER + ''), 0, None, 0 ], + 'getkq' : [0x0d, Struct(HEADER + ''), 0, None, 0 ], + 'append' : [0x0e, Struct(HEADER + ''), 0, None, None], + 'prepend' : [0x0f, Struct(HEADER + ''), 0, None, None], + 'stat' : [0x10, Struct(HEADER + ''), 0, None, 0 ], + 'setq' : [0x11, Struct(HEADER + 'LL'), 8, None, None], + 'addq' : [0x12, Struct(HEADER + 'LL'), 8, None, None], + 'replaceq' : [0x13, Struct(HEADER + 'LL'), 8, None, None], + 'deleteq' : [0x14, Struct(HEADER + ''), 0, None, 0 ], + 'incrq' : [0x15, Struct(HEADER + 'QQL'), 20, None, 0 ], + 'decrq' : [0x16, Struct(HEADER + 'QQL'), 20, None, 0 ], + 'quitq' : [0x17, Struct(HEADER + ''), 0, 0, 0 ], + 'flushq' : [0x18, Struct(HEADER + 'I'), 4, 0, 0 ], + 'appendq' : [0x19, Struct(HEADER + ''), 0, None, None], + 'prependq' : [0x1A, Struct(HEADER + ''), 0, None, None], + 'touch' : [0x1C, Struct(HEADER + 'L'), 4, None, 0 ], + 'gat' : [0x1D, Struct(HEADER + 'L'), 4, None, 0 ], + 'gatq' : [0x1E, Struct(HEADER + 'L'), 4, None, 0 ], + 'gatk' : [0x23, Struct(HEADER + 'L'), 4, None, 0 ], + 'gatkq' : [0x24, Struct(HEADER + 'L'), 4, None, 0 ], + 'sasl_list' : [0x20, Struct(HEADER + ''), 0, 0, 0 ], + 'sasl_start' : [0x21, Struct(HEADER + ''), 0, None, None], + 'sasl_step' : [0x22, Struct(HEADER + ''), 0, None, 0 ], } def is_indecrq(cmd): @@ -61,37 +66,43 @@ def is_indecrq(cmd): return True return False + +# ID: extension type, extension length, key length, val length +# key/val length: 0 means nothing must be. 'None' means should be presented ANSWERS = { 0x00: [Struct('!L'), 4, 0, None], # get - 0x01: [None, 0, 0, 0], # set - 0x02: [None, 0, 0, 0], # add - 0x03: [None, 0, 0, 0], # replace - 0x04: [None, 0, 0, 0], # delete - 0x05: [None, 0, 0, 8], # incr - 0x06: [None, 0, 0, 8], # decr - 0x07: [None, 0, 0, 0], # quit - 0x08: [None, 0, 0, 0], # flush + 0x01: [None, 0, 0, 0 ], # set + 0x02: [None, 0, 0, 0 ], # add + 0x03: [None, 0, 0, 0 ], # replace + 0x04: [None, 0, 0, 0 ], # delete + 0x05: [None, 0, 0, 8 ], # incr + 0x06: [None, 0, 0, 8 ], # decr + 0x07: [None, 0, 0, 0 ], # quit + 0x08: [None, 0, 0, 0 ], # flush 0x09: [Struct('!L'), 4, 0, None], # getq - 0x0a: [None, 0, 0, 0], # noop + 0x0a: [None, 0, 0, 0 ], # noop 0x0b: [None, 0, 0, None], # version 0x0c: [Struct('!L'), 4, None, None], # getk 0x0d: [Struct('!L'), 4, None, None], # getkq - 0x0e: [None, 0, 0, 0], # append - 0x0f: [None, 0, 0, 0], # prepend + 0x0e: [None, 0, 0, 0 ], # append + 0x0f: [None, 0, 0, 0 ], # prepend 0x10: [None, 0, None, None], # stat - 0x11: [None, 0, 0, 0], # setq - 0x12: [None, 0, 0, 0], # addq - 0x13: [None, 0, 0, 0], # replaceq - 0x14: [None, 0, 0, 0], # deleteq - 0x15: [None, 0, 0, 8], # incrq - 0x16: [None, 0, 0, 8], # decrq - 0x17: [None, 0, 0, 0], # quitq - 0x18: [None, 0, 0, 0], # flushq - 0x19: [None, 0, 0, 0], # appendq - 0x1A: [None, 0, 0, 0], # prependq - 0x1C: [None, 0, 0, 0], # touch + 0x11: [None, 0, 0, 0 ], # setq + 0x12: [None, 0, 0, 0 ], # addq + 0x13: [None, 0, 0, 0 ], # replaceq + 0x14: [None, 0, 0, 0 ], # deleteq + 0x15: [None, 0, 0, 8 ], # incrq + 0x16: [None, 0, 0, 8 ], # decrq + 0x17: [None, 0, 0, 0 ], # quitq + 0x18: [None, 0, 0, 0 ], # flushq + 0x19: [None, 0, 0, 0 ], # appendq + 0x1A: [None, 0, 0, 0 ], # prependq + 0x1C: [None, 0, 0, 0 ], # touch 0x1D: [Struct('!L'), 4, 0, None], # gat 0x1E: [Struct('!L'), 4, 0, None], # gatq + 0x20: [None, 0, 0, None], # sasl_list + 0x21: [None, 0, None, None], # sasl_start + 0x22: [None, 0, None, None], # sasl_step 0x23: [Struct('!L'), 4, None, None], # gatk 0x24: [Struct('!L'), 4, None, None], # gatkq } @@ -119,88 +130,29 @@ def is_indecrq(cmd): 'raw' : 0x00, } + class MemcachedException(Exception): def __init__(self, msg, size): self.msg = msg self.size = size -def construct_query(cmd, args): - assert(cmd in COMMANDS) - op, struct, ext_len, key_len, val_len = COMMANDS[cmd] - key = args.get('key', '') - val = args.get('val', '') - key_len = 0 if key_len is not None else len(key) - val_len = 0 if val_len is not None else len(val) - tot_len = val_len + key_len + ext_len - dtype = args.get('type', TYPE['raw']) - opaque = args.get('opaque', 0) - cas = args.get('cas', 0) - extra = args.get('extra', []) - retval = [ - struct.pack(MAGIC['request'], op, key_len, - ext_len, dtype, 0, tot_len, - opaque, cas, *extra), - key, val - ] - return ''.join(retval) - -def parse_query(cmd): - to_read = HEADER_SIZE - if len(cmd) < to_read: - raise MemcachedException("Need more bytes", to_read - len(cmd)) - - a = (magic, op, key_len, ext_len, dtype, - status, tot_len, opaque, cas) = HEADER_STRUCT.unpack_from(cmd) - to_read += tot_len - val_len = tot_len - key_len - ext_len - - if len(cmd) < to_read: - raise MemcachedException("Need more bytes", to_read - len(cmd)) - struct, ext_lenv, key_lenv, val_lenv = ANSWERS[op] - - # Multiple checks to be confident in server responses - assert(magic == MAGIC['response']) - if status == STATUS['OK']: - assert(ext_len == ext_lenv) - assert(((key_lenv > 0 or key_lenv is None) and key_len > 0) or key_len == 0) - assert(((val_lenv > 0 or val_lenv is None) and val_len > 0) or val_len == 0) - else: - assert(val_len > 0) - - retval = { - 'magic' : magic, - 'op' : op, - 'dtype' : dtype, - 'status' : status, - 'opaque' : opaque, - 'cas' : cas, - } - extra = None - if struct is not None and status == STATUS['OK']: - extra = struct.unpack_from(cmd, HEADER_SIZE) - if extra is not None: - retval['flags'] = extra[0] - - key = None - if key_lenv is None: - begin = HEADER_SIZE + ext_len - end = begin + key_len - key = cmd[begin:end] - if key is not None: - retval['key'] = key - - val = None - if val_lenv is None or val_len > 0: - begin = HEADER_SIZE + ext_len + key_len - end = HEADER_SIZE + tot_len - val = cmd[begin:end] - # decode result of (incr/decr)(q) - if is_indecrq(op): - val = INDECR_STRUCT.unpack_from(val)[0] - if val is not None: - retval['val'] = val - - return retval + +def binary_decorator(fn): + def decor(self, *args, **kwargs): + nosend = kwargs.pop('nosend', False) + result = fn(self, *args, **kwargs) + self.opaque += 1 + rv = None + if not (result is None): + rv = result + else: + self.latest = self.opaque - 1 + if nosend is False: + self.send() + rv = self.read_responses() + return rv + return decor + class MemcachedBinaryConnection(TarantoolConnection): def __init__(self, host, port): @@ -231,7 +183,64 @@ def _read_response(self): return sz def _read_and_parse_response(self): - return parse_query(self._read_response()) + cmd = self._read_response() + + to_read = HEADER_SIZE + if len(cmd) < to_read: + raise MemcachedException("Need more bytes", to_read - len(cmd)) + + a = (magic, op, key_len, ext_len, dtype, + status, tot_len, opaque, cas) = HEADER_STRUCT.unpack_from(cmd) + to_read += tot_len + val_len = tot_len - key_len - ext_len + + if len(cmd) < to_read: + raise MemcachedException("Need more bytes", to_read - len(cmd)) + struct, ext_lenv, key_lenv, val_lenv = ANSWERS[op] + + # Multiple checks to be confident in server responses + assert(magic == MAGIC['response']) + if status == STATUS['OK']: + assert(ext_len == ext_lenv) + assert(((key_lenv > 0 or key_lenv is None) and key_len > 0) or key_len == 0) + assert(((val_lenv > 0 or val_lenv is None) and val_len > 0) or val_len == 0) + else: + assert(val_len > 0) + + retval = { + 'magic' : magic, + 'op' : op, + 'dtype' : dtype, + 'status' : status, + 'opaque' : opaque, + 'cas' : cas, + } + extra = None + if struct is not None and status == STATUS['OK']: + extra = struct.unpack_from(cmd, HEADER_SIZE) + if extra is not None: + retval['flags'] = extra[0] + + key = None + if key_lenv is None: + begin = HEADER_SIZE + ext_len + end = begin + key_len + key = cmd[begin:end] + if key is not None: + retval['key'] = key + + val = None + if val_lenv is None or val_len > 0: + begin = HEADER_SIZE + ext_len + key_len + end = HEADER_SIZE + tot_len + val = cmd[begin:end] + # decode result of (incr/decr)(q) + if is_indecrq(op): + val = INDECR_STRUCT.unpack_from(val)[0] + if val is not None: + retval['val'] = val + + return retval def read_responses(self): resp = [] @@ -244,430 +253,284 @@ def read_responses(self): break return resp - def get(self, key, nosend=False): - self.commands.append(construct_query('get', { + def append_query(self, cmd, args): + assert(cmd in COMMANDS) + op, struct, ext_len, key_len, val_len = COMMANDS[cmd] + key = args.get('key', '') + val = args.get('val', '') + key_len = 0 if key_len is not None else len(key) + val_len = 0 if val_len is not None else len(val) + tot_len = val_len + key_len + ext_len + dtype = args.get('type', TYPE['raw']) + opaque = self.opaque + cas = args.get('cas', 0) + extra = args.get('extra', []) + retval = [ + struct.pack(MAGIC['request'], op, key_len, ext_len, dtype, 0, + tot_len, opaque, cas, *extra), + key, val + ] + cmd = ''.join(retval) + self.commands.append(cmd) + + @binary_decorator + def get(self, key): + self.append_query('get', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def getq(self, key, nosend=False): - self.commands.append(construct_query('getq', { + self.append_query('getq', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def getk(self, key, nosend=False): - self.commands.append(construct_query('getk', { + self.append_query('getk', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def getkq(self, key, nosend=False): - self.commands.append(construct_query('getkq', { + self.append_query('getkq', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def set(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('set', { + self.append_query('set', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def setq(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('setq', { + self.append_query('setq', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def add(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('add', { + self.append_query('add', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def addq(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('addq', { + self.append_query('addq', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def replace(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('replace', { + self.append_query('replace', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def replaceq(self, key, value, expire=0, flags=0, nosend=False, cas=-1): if (cas == -1): cas = self.cas.get(key, 0) - self.commands.append(construct_query('replaceq', { + self.append_query('replaceq', { 'key': key, 'cas': cas, 'val': value, - 'opaque': self.opaque, 'extra': [flags, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def delete(self, key, nosend=False): - self.commands.append(construct_query('delete', { + self.append_query('delete', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def deleteq(self, key, nosend=False): - self.commands.append(construct_query('deleteq', { + self.append_query('deleteq', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def incr(self, key, delta=1, expire=0xFFFFFFFF, initial=0, nosend=False): - self.commands.append(construct_query('incr', { + self.append_query('incr', { 'key': key, - 'opaque': self.opaque, 'extra': [delta, initial, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def decr(self, key, delta=1, expire=0xFFFFFFFF, initial=0, nosend=False): - self.commands.append(construct_query('decr', { + self.append_query('decr', { 'key': key, - 'opaque': self.opaque, 'extra': [delta, initial, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def incrq(self, key, delta=1, expire=0xFFFFFFFF, initial=0, nosend=False): - self.commands.append(construct_query('incrq', { + self.append_query('incrq', { 'key': key, - 'opaque': self.opaque, 'extra': [delta, initial, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def decrq(self, key, delta=1, expire=0xFFFFFFFF, initial=0, nosend=False): - self.commands.append(construct_query('decrq', { + self.append_query('decrq', { 'key': key, - 'opaque': self.opaque, 'extra': [delta, initial, expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def quit(self, nosend=False): - self.commands.append(construct_query('quit', { - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + self.append_query('quit', {}) + @binary_decorator def quitq(self, nosend=False): - self.commands.append(construct_query('quitq', { - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + self.append_query('quitq', {}) + @binary_decorator def flush(self, expire=0, nosend=False): - self.commands.append(construct_query('flush', { - 'opaque': self.opaque, + self.append_query('flush', { 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def flushq(self, expire=0, nosend=False): - self.commands.append(construct_query('flushq', { - 'opaque': self.opaque, + self.append_query('flushq', { 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def noop(self): - self.commands.append(construct_query('noop', { - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 + self.append_query('noop', {}) + self.latest = self.opaque self.send() return self.read_responses() + @binary_decorator def version(self, nosend=False): - self.commands.append(construct_query('version', { - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + self.append_query('version', {}) + @binary_decorator def append(self, key, value, nosend=False): - self.commands.append(construct_query('append', { + self.append_query('append', { 'key': key, 'val': value, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def prepend(self, key, value, nosend=False): - self.commands.append(construct_query('prepend', { + self.append_query('prepend', { 'key': key, 'val': value, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def appendq(self, key, value, nosend=False): - self.commands.append(construct_query('appendq', { + self.append_query('appendq', { 'key': key, 'val': value, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def prependq(self, key, value, nosend=False): - self.commands.append(construct_query('prependq', { + self.append_query('prependq', { 'key': key, 'val': value, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + + @binary_decorator def stat(self, key=''): - self.commands.append(construct_query('stat', { + # nosend flag is ignored + self.append_query('stat', { 'key': key, - 'opaque': self.opaque, - })) - self.latest = self.opaque - self.opaque += 1 - ans = {} + }) + self.latest = self.opaque self.send() + ans = {} while True: rv = self._read_and_parse_response() if 'key' in rv and not rv['key'] and \ 'val' in rv and not rv['val']: return ans ans[rv['key']] = rv['val'] + return ans + @binary_decorator def touch(self, key, expire, nosend=False): - self.commands.append(construct_query('touch', { + self.append_query('touch', { 'key': key, - 'opaque': self.opaque, 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def gat(self, key, expire, nosend=False): - self.commands.append(construct_query('gat', { + self.append_query('gat', { 'key': key, - 'opaque': self.opaque, 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def gatk(self, key, expire, nosend=False): - self.commands.append(construct_query('gatk', { + self.append_query('gatk', { 'key': key, - 'opaque': self.opaque, 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def gatq(self, key, expire, nosend=False): - self.commands.append(construct_query('gat', { + self.append_query('gat', { 'key': key, - 'opaque': self.opaque, 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + @binary_decorator def gatkq(self, key, expire, nosend=False): - self.commands.append(construct_query('gatk', { + self.append_query('gatk', { 'key': key, - 'opaque': self.opaque, 'extra': [expire] - })) - self.latest = self.opaque - self.opaque += 1 - if not nosend: - self.send() - return self.read_responses() - return None + }) + + @binary_decorator + def sasl_list(self): + self.append_query('sasl_list', {}) + + @binary_decorator + def sasl_start(self, mech, value): + self.append_query('sasl_start', { + 'key': mech, + 'val': value + }) + + @binary_decorator + def sasl_step(self, value): + self.append_query('sasl_step', { + 'val': value + }) MEMCACHED_SEPARATOR = '\r\n' diff --git a/test/sasl/binary-sasl.result b/test/sasl/binary-sasl.result new file mode 100644 index 0000000..680246a --- /dev/null +++ b/test/sasl/binary-sasl.result @@ -0,0 +1 @@ +#---------------------------# sasl tests #--------------------------# diff --git a/test/sasl/binary-sasl.test.py b/test/sasl/binary-sasl.test.py new file mode 100644 index 0000000..1dc28e6 --- /dev/null +++ b/test/sasl/binary-sasl.test.py @@ -0,0 +1,148 @@ +import os +import sys +import inspect +import traceback + +saved_path = sys.path[:] +sys.path.append(os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda:0)))) + +from internal.memcached_connection import MemcachedBinaryConnection +from internal.memcached_connection import STATUS, COMMANDS + +mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + +def iequal(left, right, level = 1): + if (left != right): + tb = traceback.extract_stack()[-(level + 1)] + print "Error on line %s:%d: %s not equal %s" % (tb[0], tb[1], + repr(left), repr(right)) + return False + return True + +def inequal(left, right, level = 0): + if (left == right): + tb = traceback.extract_stack()[-(level + 1)] + print "Error on line %s:%d: %s equal %s" % (tb[0], tb[1], + repr(left), repr(right)) + return False + return True + +def issert(stmt, level = 0): + if bool(stmt): + tb = traceback.extract_stack()[-(level + 1)] + print "Error on line %s:%d: is False" % (tb[0], tb[1], + repr(left), repr(right)) + return False + return True + +def __check(res, flags, val, level = 0): + return iequal(res.get('flags', -1), flags, level + 1) and \ + iequal(res.get('val', val), val, level + 1) + +def check(key, flags, val, level = 0): + res = mc.get(key) + __check(res[0], flags, val, level + 1) + +def set(key, expire, flags, value): + res = mc.set(key, value, expire, flags) + return check(key, flags, value, 1) + +def empty(key): + res = mc.get(key) + return iequal(res[0]['status'], STATUS['KEY_ENOENT'], 1) + +def delete(key, when): + res = mc.delete(key) + empty(key) + +print("""#---------------------------# sasl tests #--------------------------#""") + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + res = mc1.set('x', 'somevalue') + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + res = mc1.delete('x', 'somevalue') + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + res = mc1.set('x', 'somevalue') + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + res = mc1.flush() + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +res = mc.sasl_list() +if res[0]['val'].find('PLAIN') == -1: + print "Error" + +secret = '%c%s%c%s' % (0, 'testuser', 0, 'testpass') +res = mc.sasl_start("X" * 40, secret) +iequal(res[0]['status'], STATUS['AUTH_ERROR']) +# mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + +secret = '%c%s%c%s' % (0, 'testuser', 0, 'badpass') +res = mc.sasl_start("PLAIN", secret) +iequal(res[0]['status'], STATUS['AUTH_ERROR']) +# mc = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + +secret = '%c%s%c%s' % (0, 'testuser', 0, 'testpass') +res = mc.sasl_start("PLAIN", secret) +iequal(res[0]['status'], 0) + +if True: + empty('x') + res = mc.set('x', 'somevalue') + iequal(res[0]['status'], 0) + +if True: + check('x', 0, 'somevalue') + res = mc.delete('x', 'somevalue') + iequal(res[0]['status'], 0) + +if True: + empty('x') + res = mc.set('x', 'somevalue') + iequal(res[0]['status'], 0) + +if True: + check('x', 0, 'somevalue') + mc.flush() + empty('x') + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + secret = '%c%s%c%s' % (0, 'testuser', 0, 'wrongpass') + res = mc1.sasl_start('PLAIN', secret) + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + + res = mc1.set('x', 'somevalue') + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +if True: + mc1 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + secret = '%c%s%c%s' % (0, 'testuser', 0, 'wrongpass') + res = mc1.sasl_start('PLAIN', secret) + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + + mc2 = MemcachedBinaryConnection("127.0.0.1", iproto.py_con.port) + secret = '%c%s%c%s' % (0, 'testuser', 0, 'testpass') + res = mc2.sasl_start('PLAIN', secret) + iequal(res[0]['status'], 0) + res = mc2.set('x', 'somevalue') + iequal(res[0]['status'], 0) + + res = mc1.set('x', 'somevalue') + iequal(res[0]['status'], STATUS['AUTH_ERROR']) + +if True: + res = mc.stat() + iequal(res['auth_cmds'], '6') + iequal(res['auth_errors'], '4') + +sys.path = saved_path diff --git a/test/sasl/config/tarantool-memcached.conf b/test/sasl/config/tarantool-memcached.conf new file mode 100644 index 0000000..ab62caf --- /dev/null +++ b/test/sasl/config/tarantool-memcached.conf @@ -0,0 +1,3 @@ +mech_list: plain cram-md5 +log_level: 20 +sasldb_path: /tmp/test-tarantool-memcached.sasldb diff --git a/test/sasl/internal b/test/sasl/internal new file mode 120000 index 0000000..ac23b96 --- /dev/null +++ b/test/sasl/internal @@ -0,0 +1 @@ +../internal/ \ No newline at end of file diff --git a/test/sasl/sasl.lua b/test/sasl/sasl.lua new file mode 100644 index 0000000..fb65dcb --- /dev/null +++ b/test/sasl/sasl.lua @@ -0,0 +1,83 @@ +#!/usr/bin/env tarantool + +package.cpath = './?.so;' .. package.cpath +-------------------------------------------------------------------------------- +-- Manipulating environment variables -- +-------------------------------------------------------------------------------- +local ffi = require('ffi') +local log = require('log') +local errno = require('errno') + +ffi.cdef[[ + extern char **environ; + + int setenv(const char *name, const char *value, int overwrite); + int unsetenv(const char *name); + int clearenv(void); + char *getenv(const char *name); +]] + +local env = setmetatable({}, { + __call = function() + local environ = ffi.C.environ + if not environ then + return nil + end + local r = {} + local i = 0 + while environ[i] ~= 0 do + local e = ffi.string(environ[i]) + local eq = e:find('=') + if eq then + r[e:sub(1, eq - 1)] = e:sub(eq + 1) + end + i = i + 1 + end + return r + end, + __index = function(self, key) + local var = ffi.C.getenv(key) + if var == 0 then + return nil + end + return ffi.string(var) + end, + __newindex = function(self, key, value) + local rv = nil + if value ~= nil then + rv = ffi.C.setenv(key, value, 1) + else + rv = ffi.C.unsetenv(key) + end + if rv == -1 then + error(string.format('error %d: %s', errno(), errno.errstring())) + end + end +}) + +-------------------------------------------------------------------------------- + +local fio = require('fio') + +local sasldb = '/tmp/test-tarantool-memcached.sasldb' +fio.unlink(sasldb) +os.execute('echo testpass | saslpasswd2 -a tarantool-memcached -c -p testuser -f ' .. sasldb) + +env['SASL_CONF_PATH'] = fio.pathjoin(fio.cwd(), '../sasl/config/') + +box.cfg{ + wal_mode = 'none', + slab_alloc_arena = 0.1, +} + +local memcached = require('memcached') + +local listen_port = env['LISTEN']:match(':(.*)') +local admin_port = env['ADMIN'] + +local inst = memcached.create('memcached', listen_port, { + expire_full_scan_time = 1, + sasl = true +}):grant('guest') + +require('console').listen(admin_port) diff --git a/test/sasl/suite.ini b/test/sasl/suite.ini new file mode 100644 index 0000000..437ea1b --- /dev/null +++ b/test/sasl/suite.ini @@ -0,0 +1,4 @@ +[default] +core = tarantool +script = sasl.lua +description = memcached binary sasl tests