From 520ea27dfc0d9a425e0eca4ea19c9290a01e4202 Mon Sep 17 00:00:00 2001 From: Ellis Breen Date: Thu, 21 Dec 2017 19:09:17 +0000 Subject: [PATCH] PYCBC-412: update SDK to provide diagnostics/ping support Motivation ---------- The libcouchbase SDK has evolved in how it provides diagnostics pertinent to the health of the cluster. Python SDK must be updated to reflect these changes. Modifications ------------- - Renamed Bucket.get_health to Bucket.ping (to match lcb_ping3) from libcouchbase, to better represent the abstraction level provided. Users will have to build their own heuristics that interpret the information returned from ping. - Exposed the lcb_diag function as Bucket.diag, providing activity information, connection status per node, as well as API/version information from the client. Results ------- Client now successfully passes through the diag and ping result structures, as validated by the schema listed in "couchbase.tests.cases.diag_t". Change-Id: I15909dca1c12d8e79f9f76bc3419f53ec5424b1a Reviewed-on: http://review.couchbase.org/87160 Reviewed-by: Sergey Avseyev Tested-by: Ellis Breen --- .gitignore | 3 + CMakeLists.txt | 35 +++++---- couchbase/bucket.py | 47 ++++++++++-- couchbase/tests/cases/diag_t.py | 117 ++++++++++++++++++++++++++++++ couchbase/tests/cases/health_t.py | 70 ------------------ src/bucket.c | 3 +- src/callbacks.c | 37 ++++++++++ src/miscops.c | 45 +++++++++++- src/pycbc.h | 10 ++- 9 files changed, 269 insertions(+), 98 deletions(-) create mode 100644 couchbase/tests/cases/diag_t.py delete mode 100644 couchbase/tests/cases/health_t.py diff --git a/.gitignore b/.gitignore index f39f3a055..2c67a44bd 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ CouchbaseMock.jar /.pydevproject /cmake-build-debug/ /make.sh +html +latex + diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f94ee3df..76f520e60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,30 @@ cmake_minimum_required(VERSION 3.8) project(couchbase_python_client_2_3_1) + +set(LCB_ROOT ../libcouchbase) + +set(PYTHON_INCLUDE_DIR /Users/ellis_breen/root/virtualenvs/3.6/default/include/python3.6m/) +set(PYTHON_INCLUDE ${PYTHON_INCLUDE_DIR}) +aux_source_directory(${LCB_ROOT}/src LCB_CORE) +aux_source_directory(${LCB_ROOT}/src/http HTTP) +aux_source_directory(${LCB_ROOT}/include/memcached/ MCD_INC) +aux_source_directory(${LCB_ROOT}/include/libcouchbase/ LCB_INC) +aux_source_directory(${LCB_ROOT}/contrib/lcb-jsoncpp LCB_JSON) + +include_directories( ${LCB_CORE} + ${LCB_ROOT}/include + ${MCD_INC} + ${HTTP} + ${PYTHON_INCLUDE_DIR} + ${LCB_JSON} + ) include(ExternalProject) ExternalProject_Add(libcouchbase DOWNLOAD_COMMAND "" - SOURCE_DIR ${../libcouchbase} + SOURCE_DIR ${LCB_ROOT} ) set(CMAKE_CXX_STANDARD 11) -set(LCB_ROOT ../libcouchbase) set(EXTRA_SOURCE_DIRS ${LCB_ROOT}/src ${LCB_ROOT}/include/memcached @@ -15,11 +32,6 @@ set(EXTRA_SOURCE_DIRS ${LCB_ROOT}/src/http ) -aux_source_directory(../libcouchbase/src LCB_CORE) -aux_source_directory(../libcouchbase/src/http HTTP) -aux_source_directory(../libcouchbase/include/memcached/ MCD_INC) -aux_source_directory(../libcouchbase/include/libcouchbase/ LCB_INC) - set(SOURCE ${SOURCE} ${LCB_CORE} @@ -27,18 +39,15 @@ set(SOURCE ${MCD_INC} ${HTTP} ) -link_directories(../libcouchbase/lib) -link_libraries(../libcouchbase/lib/libcouchbase.dylib +link_directories(${LCB_ROOT}/lib) +link_libraries(${LCB_ROOT}/../lib/ ) -include_directories(${LCB_CORE} - ${LCB_INC} - ${MCD_INC} - ${HTTP}) add_executable(couchbase_python_client_2_3_1 ${EXTRA_SOURCE_DIRS} ${LCB_CORE} ${LCB_INC} ${MCD_INC} + ${LCB_JSON} acouchbase/tests/asyncio_tests.py acouchbase/tests/fixtures.py acouchbase/tests/py34only.py diff --git a/couchbase/bucket.py b/couchbase/bucket.py index 9d7e63f8d..5aaf8fdc5 100644 --- a/couchbase/bucket.py +++ b/couchbase/bucket.py @@ -33,6 +33,7 @@ from couchbase._pyport import basestring import couchbase.subdocument as SD import couchbase.priv_constants as _P +import json ### Private constants. This is to avoid imposing a dependency requirement ### For simple flags: @@ -908,26 +909,56 @@ def stats(self, keys=None, keystats=False): keys = (keys,) return self._stats(keys, keystats=keystats) - def get_health(self): - """Request cluster health information. + def ping(self): + """Ping cluster for latency/status information per-service - Fetches health information from each node in the cluster. - It returns a `dict` with 'type' keys - and server summary lists as a value. + Pings each node in the cluster, and + returns a `dict` with 'type' keys (e.g 'n1ql', 'kv') + and node service summary lists as a value. :raise: :exc:`.CouchbaseNetworkError` :return: `dict` where keys are stat keys and values are host-value pairs - Get health info (works on couchbase buckets):: + Ping cluster (works on couchbase buckets):: - cb.get_health() + cb.ping() # {'services': {...}, ...} """ - resultdict = self._get_health() + resultdict = self._ping() return resultdict['services_struct'] + def diagnostics(self): + """Request diagnostics report about network connections + + Generates diagnostics for each node in the cluster. + It returns a `dict` with details + + + :raise: :exc:`.CouchbaseNetworkError` + :return: `dict` where keys are stat keys and values are + host-value pairs + + Get health info (works on couchbase buckets):: + + cb.diagnostics() + # { + 'config': + { + 'id': node ID, + 'last_activity_us': time since last activity in nanoseconds + 'local': local server and port, + 'remote': remote server and port, + 'status': connection status + } + 'id': client ID, + 'sdk': sdk version, + 'version': diagnostics API version + } + """ + return json.loads(self._diagnostics()['health_json']) + def observe(self, key, master_only=False): """Return storage information for a key. diff --git a/couchbase/tests/cases/diag_t.py b/couchbase/tests/cases/diag_t.py new file mode 100644 index 000000000..96c6086bc --- /dev/null +++ b/couchbase/tests/cases/diag_t.py @@ -0,0 +1,117 @@ +# +# Copyright 2017, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from unittest import SkipTest + +from couchbase.tests.base import ConnectionTestCase +import jsonschema +import re + +# For Python 2/3 compatibility +try: + basestring +except NameError: + basestring = str + +service_schema = {"type": "object", + "properties": {"details": {"type": "string"}, + "latency": {"anyOf": [{"type": "number"}, {"type": "string"}]}, + "server": {"type": "string"}, + "status": {"type": "number"} + }, + "required": ["details", "latency", "server", "status"]} + +any_of_required_services_schema = {"type": "array", + "items": service_schema} + + +def gen_schema_for_services_with_required_entry(name): + return {"type": "object", + "properties": {name: any_of_required_services_schema}, + "required": [name] + } + + +any_of_required_services_schema = {"anyOf": + [gen_schema_for_services_with_required_entry(name) for name in ["n1ql", "views", "fts", "kv"]] + } + +ping_schema = {"anyOf": [{ + "type": "object", + "properties": { + "services": any_of_required_services_schema + }, + "required": ["services"] +}]} + +server_and_port_schema = {"type": "string", + "pattern": "([0-9]{1,3}\.){3,3}[0-9]{1,3}:[0-9]+"} +connection_status_schema = {"type": "string", + "pattern": "connected"} +config_schema = {"type": "array", + "items": {"type": "object", + "properties": { + "id": {"type": "string"}, + "last_activity_us": {"type": "number"}, + "local": server_and_port_schema, + "remote": server_and_port_schema, + "status": connection_status_schema + }}} + +python_id="PYCBC" + +client_id_schema = {"type": "string", + "pattern": "^0x[a-f0-9]+/"+python_id} + +three_part_ver_num = "([0-9]+\.)+[0-9]+" + +sdk_schema = {"type": "string", + "pattern": "libcouchbase" + + re.escape("/") + three_part_ver_num + "_[0-9]+_(.*?)" + + re.escape(python_id + "/") + + three_part_ver_num + "\.[^\s]*"} + + +diagnostics_schema = {"type": "object", + "properties": { + "config": config_schema, + "id": client_id_schema, + "sdk": sdk_schema, + "version": {"type": "number"} + + }} + + +class DiagnosticsTests(ConnectionTestCase): + + def setUp(self): + super(DiagnosticsTests, self).setUp() + + def test_ping(self): + result = self.cb.ping() + jsonschema.validate(result, any_of_required_services_schema) + + def test_diagnostics(self): + + if self.is_mock: + raise SkipTest() + result = self.cb.diagnostics() + + jsonschema.validate(result, diagnostics_schema) + + +if __name__ == '__main__': + unittest.main() diff --git a/couchbase/tests/cases/health_t.py b/couchbase/tests/cases/health_t.py deleted file mode 100644 index 970fb1c6a..000000000 --- a/couchbase/tests/cases/health_t.py +++ /dev/null @@ -1,70 +0,0 @@ -# -# Copyright 2017, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase -import jsonschema - -# For Python 2/3 compatibility -try: - basestring -except NameError: - basestring = str - - -class HealthTest(ConnectionTestCase): - def setUp(self): - super(HealthTest, self).setUp() - self.skipUnlessMock() - - server_schema = {"type": "object", - "properties": {"details": {"type": "string"}, - "latency": {"anyOf": [{"type": "number"}, {"type": "string"}]}, - "server": {"type": "string"}, - "status": {"type": "number"} - }, - "required": ["details", "latency", "server", "status"]} - - servers_schema = {"type": "array", - "items": server_schema} - - @staticmethod - def gen_schema(name): - return {"type": "object", - "properties": {name: HealthTest.servers_schema}, - "required": [name] - } - - def test_health(self): - result = self.cb.get_health() - - services_schema = {"anyOf": - [HealthTest.gen_schema(name) for name in ["n1ql", "views", "fts", "kv"]] - } - - health_schema = {"anyOf": [{ - "type": "object", - "properties": { - "services": services_schema - }, - "required": ["services"] - }]} - - jsonschema.validate(result, services_schema) - - -if __name__ == '__main__': - unittest.main() diff --git a/src/bucket.c b/src/bucket.c index bd980b99a..430773613 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -513,7 +513,8 @@ static PyMethodDef Bucket_TABLE_methods[] = { OPFUNC(counter, "Modify a counter in Couchbase"), OPFUNC(counter_multi, "Multi-key variant of counter"), OPFUNC(_stats, "Get various server statistics"), - OPFUNC(_get_health, "Get health info"), + OPFUNC(_ping, "Ping cluster to receive diagnostics"), + OPFUNC(_diagnostics, "Get diagnostics"), OPFUNC(_http_request, "Internal routine for HTTP requests"), OPFUNC(_view_request, "Internal routine for view requests"), diff --git a/src/callbacks.c b/src/callbacks.c index 360056a94..114a27ee6 100644 --- a/src/callbacks.c +++ b/src/callbacks.c @@ -799,6 +799,42 @@ static void ping_callback(lcb_t instance, CB_THR_BEGIN(parent); } + +static void diag_callback(lcb_t instance, + int cbtype, + const lcb_RESPBASE *resp_base) +{ + pycbc_Bucket *parent; + const lcb_RESPDIAG *resp = (const lcb_RESPDIAG *)resp_base; + + pycbc_MultiResult *mres = (pycbc_MultiResult *)resp->cookie; + PyObject *resultdict = pycbc_multiresult_dict(mres); + parent = mres->parent; + CB_THR_END(parent); + + if (resp->rc != LCB_SUCCESS) { + if (mres->errop == NULL) { + pycbc_Result *res = (pycbc_Result *)pycbc_result_new(parent); + res->rc = resp->rc; + res->key = Py_None; + Py_INCREF(res->key); + maybe_push_operr(mres, res, resp->rc, 0); + } + } + + if (resp->njson) { + pycbc_dict_add_text_kv(resultdict, "health_json", resp->json); + } + if (resp->rflags & LCB_RESP_F_FINAL) { + /* Note this can happen in both success and error cases!*/ + operation_completed_with_err_info(parent, mres, cbtype, resp_base); + } + + CB_THR_BEGIN(parent); +} + + + void pycbc_callbacks_init(lcb_t instance) { @@ -813,6 +849,7 @@ pycbc_callbacks_init(lcb_t instance) lcb_install_callback3(instance, LCB_CALLBACK_OBSERVE, observe_callback); lcb_install_callback3(instance, LCB_CALLBACK_STATS, stats_callback); lcb_install_callback3(instance, LCB_CALLBACK_PING, ping_callback); + lcb_install_callback3(instance, LCB_CALLBACK_DIAG, diag_callback); /* Subdoc */ lcb_install_callback3(instance, LCB_CALLBACK_SDLOOKUP, subdoc_callback); diff --git a/src/miscops.c b/src/miscops.c index 06343a1b8..cc1d3bdb3 100644 --- a/src/miscops.c +++ b/src/miscops.c @@ -14,7 +14,9 @@ * limitations under the License. **/ +#include #include "oputil.h" +#include "pycbc.h" /** * This file contains 'miscellaneous' operations. Functions contained here @@ -361,9 +363,9 @@ pycbc_Bucket__stats(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) return cv.ret; } -PyObject *pycbc_Bucket__get_health(pycbc_Bucket *self, - PyObject *args, - PyObject *kwargs) +PyObject *pycbc_Bucket__ping(pycbc_Bucket *self, + PyObject *args, + PyObject *kwargs) { int rv; Py_ssize_t ncmds = 0; @@ -397,3 +399,40 @@ PyObject *pycbc_Bucket__get_health(pycbc_Bucket *self, pycbc_common_vars_finalize(&cv, self); return cv.ret; } + +PyObject *pycbc_Bucket__diagnostics(pycbc_Bucket *self, + PyObject *args, + PyObject *kwargs) +{ + int rv; + Py_ssize_t ncmds = 0; + lcb_error_t err = LCB_ERROR; + struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; + lcb_CMDDIAG cmd = {0}; + cmd.options = LCB_PINGOPT_F_JSONPRETTY; + + cmd.id = "PYCBC"; + rv = pycbc_common_vars_init(&cv, self, PYCBC_ARGOPT_MULTI, ncmds, 0); + + if (rv < 0) { + return NULL; + } + lcb_sched_enter(self->instance); + PYCBC_CONN_THR_BEGIN(self); + err = lcb_diag(self->instance, cv.mres, &cmd); + PYCBC_CONN_THR_END(self); + + if (err != LCB_SUCCESS) { + PYCBC_EXCTHROW_SCHED(err); + goto GT_DONE; + } + + if (-1 == pycbc_common_vars_wait(&cv, self)) { + goto GT_DONE; + } + lcb_sched_leave(self->instance); + + GT_DONE: + pycbc_common_vars_finalize(&cv, self); + return cv.ret; +} \ No newline at end of file diff --git a/src/pycbc.h b/src/pycbc.h index e99e58641..a15479387 100644 --- a/src/pycbc.h +++ b/src/pycbc.h @@ -1127,9 +1127,13 @@ PyObject* pycbc_Bucket__cntlstr(pycbc_Bucket *conn, PyObject *args, PyObject *kw /** * Health-check methods */ -PyObject *pycbc_Bucket__get_health(pycbc_Bucket *self, - PyObject *args, - PyObject *kwargs); +PyObject *pycbc_Bucket__ping(pycbc_Bucket *self, + PyObject *args, + PyObject *kwargs); + +PyObject *pycbc_Bucket__diagnostics(pycbc_Bucket *self, + PyObject *args, + PyObject *kwargs); /** * Flag to check if logging is enabled for the library via Python's logging */