Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
PYCBC-412: update SDK to provide diagnostics/ping support
Motivation
----------
The libcouchbase SDK has evolved in how it provides
diagnostics pertinent to the health of the cluster.
Python SDK must be updated to reflect these changes.

Modifications
-------------
- Renamed Bucket.get_health to Bucket.ping (to match lcb_ping3)
from libcouchbase, to better represent the abstraction level
provided. Users will have to build their own heuristics that
interpret the information returned from ping.
- Exposed the lcb_diag function as Bucket.diag, providing
activity information, connection status per node,
as well as API/version information from the client.

Results
-------
Client now successfully passes through the diag and
ping result structures, as validated by the schema listed
in "couchbase.tests.cases.diag_t".

Change-Id: I15909dca1c12d8e79f9f76bc3419f53ec5424b1a
Reviewed-on: http://review.couchbase.org/87160
Reviewed-by: Sergey Avseyev <sergey.avseyev@gmail.com>
Tested-by: Ellis Breen <ellis.breen@couchbase.com>
  • Loading branch information
griels committed Jan 9, 2018
1 parent 275f0b9 commit 520ea27
Show file tree
Hide file tree
Showing 9 changed files with 269 additions and 98 deletions.
3 changes: 3 additions & 0 deletions .gitignore
Expand Up @@ -38,3 +38,6 @@ CouchbaseMock.jar
/.pydevproject
/cmake-build-debug/
/make.sh
html
latex

35 changes: 22 additions & 13 deletions CMakeLists.txt
@@ -1,44 +1,53 @@
cmake_minimum_required(VERSION 3.8)
project(couchbase_python_client_2_3_1)

set(LCB_ROOT ../libcouchbase)

set(PYTHON_INCLUDE_DIR /Users/ellis_breen/root/virtualenvs/3.6/default/include/python3.6m/)
set(PYTHON_INCLUDE ${PYTHON_INCLUDE_DIR})
aux_source_directory(${LCB_ROOT}/src LCB_CORE)
aux_source_directory(${LCB_ROOT}/src/http HTTP)
aux_source_directory(${LCB_ROOT}/include/memcached/ MCD_INC)
aux_source_directory(${LCB_ROOT}/include/libcouchbase/ LCB_INC)
aux_source_directory(${LCB_ROOT}/contrib/lcb-jsoncpp LCB_JSON)

include_directories( ${LCB_CORE}
${LCB_ROOT}/include
${MCD_INC}
${HTTP}
${PYTHON_INCLUDE_DIR}
${LCB_JSON}
)
include(ExternalProject)

ExternalProject_Add(libcouchbase
DOWNLOAD_COMMAND ""
SOURCE_DIR ${../libcouchbase}
SOURCE_DIR ${LCB_ROOT}
)
set(CMAKE_CXX_STANDARD 11)
set(LCB_ROOT ../libcouchbase)
set(EXTRA_SOURCE_DIRS
${LCB_ROOT}/src
${LCB_ROOT}/include/memcached
${LCB_ROOT}/include/libcouchbase
${LCB_ROOT}/src/http
)

aux_source_directory(../libcouchbase/src LCB_CORE)
aux_source_directory(../libcouchbase/src/http HTTP)
aux_source_directory(../libcouchbase/include/memcached/ MCD_INC)
aux_source_directory(../libcouchbase/include/libcouchbase/ LCB_INC)

set(SOURCE
${SOURCE}
${LCB_CORE}
${LCB_INC}
${MCD_INC}
${HTTP}
)
link_directories(../libcouchbase/lib)
link_libraries(../libcouchbase/lib/libcouchbase.dylib
link_directories(${LCB_ROOT}/lib)
link_libraries(${LCB_ROOT}/../lib/
)
include_directories(${LCB_CORE}
${LCB_INC}
${MCD_INC}
${HTTP})
add_executable(couchbase_python_client_2_3_1
${EXTRA_SOURCE_DIRS}
${LCB_CORE}
${LCB_INC}
${MCD_INC}
${LCB_JSON}
acouchbase/tests/asyncio_tests.py
acouchbase/tests/fixtures.py
acouchbase/tests/py34only.py
Expand Down
47 changes: 39 additions & 8 deletions couchbase/bucket.py
Expand Up @@ -33,6 +33,7 @@
from couchbase._pyport import basestring
import couchbase.subdocument as SD
import couchbase.priv_constants as _P
import json

### Private constants. This is to avoid imposing a dependency requirement
### For simple flags:
Expand Down Expand Up @@ -908,26 +909,56 @@ def stats(self, keys=None, keystats=False):
keys = (keys,)
return self._stats(keys, keystats=keystats)

def get_health(self):
"""Request cluster health information.
def ping(self):
"""Ping cluster for latency/status information per-service
Fetches health information from each node in the cluster.
It returns a `dict` with 'type' keys
and server summary lists as a value.
Pings each node in the cluster, and
returns a `dict` with 'type' keys (e.g 'n1ql', 'kv')
and node service summary lists as a value.
:raise: :exc:`.CouchbaseNetworkError`
:return: `dict` where keys are stat keys and values are
host-value pairs
Get health info (works on couchbase buckets)::
Ping cluster (works on couchbase buckets)::
cb.get_health()
cb.ping()
# {'services': {...}, ...}
"""
resultdict = self._get_health()
resultdict = self._ping()
return resultdict['services_struct']

def diagnostics(self):
"""Request diagnostics report about network connections
Generates diagnostics for each node in the cluster.
It returns a `dict` with details
:raise: :exc:`.CouchbaseNetworkError`
:return: `dict` where keys are stat keys and values are
host-value pairs
Get health info (works on couchbase buckets)::
cb.diagnostics()
# {
'config':
{
'id': node ID,
'last_activity_us': time since last activity in nanoseconds
'local': local server and port,
'remote': remote server and port,
'status': connection status
}
'id': client ID,
'sdk': sdk version,
'version': diagnostics API version
}
"""
return json.loads(self._diagnostics()['health_json'])

def observe(self, key, master_only=False):
"""Return storage information for a key.
Expand Down
117 changes: 117 additions & 0 deletions couchbase/tests/cases/diag_t.py
@@ -0,0 +1,117 @@
#
# Copyright 2017, Couchbase, Inc.
# All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License")
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from unittest import SkipTest

from couchbase.tests.base import ConnectionTestCase
import jsonschema
import re

# For Python 2/3 compatibility
try:
basestring
except NameError:
basestring = str

service_schema = {"type": "object",
"properties": {"details": {"type": "string"},
"latency": {"anyOf": [{"type": "number"}, {"type": "string"}]},
"server": {"type": "string"},
"status": {"type": "number"}
},
"required": ["details", "latency", "server", "status"]}

any_of_required_services_schema = {"type": "array",
"items": service_schema}


def gen_schema_for_services_with_required_entry(name):
return {"type": "object",
"properties": {name: any_of_required_services_schema},
"required": [name]
}


any_of_required_services_schema = {"anyOf":
[gen_schema_for_services_with_required_entry(name) for name in ["n1ql", "views", "fts", "kv"]]
}

ping_schema = {"anyOf": [{
"type": "object",
"properties": {
"services": any_of_required_services_schema
},
"required": ["services"]
}]}

server_and_port_schema = {"type": "string",
"pattern": "([0-9]{1,3}\.){3,3}[0-9]{1,3}:[0-9]+"}
connection_status_schema = {"type": "string",
"pattern": "connected"}
config_schema = {"type": "array",
"items": {"type": "object",
"properties": {
"id": {"type": "string"},
"last_activity_us": {"type": "number"},
"local": server_and_port_schema,
"remote": server_and_port_schema,
"status": connection_status_schema
}}}

python_id="PYCBC"

client_id_schema = {"type": "string",
"pattern": "^0x[a-f0-9]+/"+python_id}

three_part_ver_num = "([0-9]+\.)+[0-9]+"

sdk_schema = {"type": "string",
"pattern": "libcouchbase" +
re.escape("/") + three_part_ver_num + "_[0-9]+_(.*?)" +
re.escape(python_id + "/") +
three_part_ver_num + "\.[^\s]*"}


diagnostics_schema = {"type": "object",
"properties": {
"config": config_schema,
"id": client_id_schema,
"sdk": sdk_schema,
"version": {"type": "number"}

}}


class DiagnosticsTests(ConnectionTestCase):

def setUp(self):
super(DiagnosticsTests, self).setUp()

def test_ping(self):
result = self.cb.ping()
jsonschema.validate(result, any_of_required_services_schema)

def test_diagnostics(self):

if self.is_mock:
raise SkipTest()
result = self.cb.diagnostics()

jsonschema.validate(result, diagnostics_schema)


if __name__ == '__main__':
unittest.main()
70 changes: 0 additions & 70 deletions couchbase/tests/cases/health_t.py

This file was deleted.

3 changes: 2 additions & 1 deletion src/bucket.c
Expand Up @@ -513,7 +513,8 @@ static PyMethodDef Bucket_TABLE_methods[] = {
OPFUNC(counter, "Modify a counter in Couchbase"),
OPFUNC(counter_multi, "Multi-key variant of counter"),
OPFUNC(_stats, "Get various server statistics"),
OPFUNC(_get_health, "Get health info"),
OPFUNC(_ping, "Ping cluster to receive diagnostics"),
OPFUNC(_diagnostics, "Get diagnostics"),

OPFUNC(_http_request, "Internal routine for HTTP requests"),
OPFUNC(_view_request, "Internal routine for view requests"),
Expand Down
37 changes: 37 additions & 0 deletions src/callbacks.c
Expand Up @@ -799,6 +799,42 @@ static void ping_callback(lcb_t instance,
CB_THR_BEGIN(parent);
}


static void diag_callback(lcb_t instance,
int cbtype,
const lcb_RESPBASE *resp_base)
{
pycbc_Bucket *parent;
const lcb_RESPDIAG *resp = (const lcb_RESPDIAG *)resp_base;

pycbc_MultiResult *mres = (pycbc_MultiResult *)resp->cookie;
PyObject *resultdict = pycbc_multiresult_dict(mres);
parent = mres->parent;
CB_THR_END(parent);

if (resp->rc != LCB_SUCCESS) {
if (mres->errop == NULL) {
pycbc_Result *res = (pycbc_Result *)pycbc_result_new(parent);
res->rc = resp->rc;
res->key = Py_None;
Py_INCREF(res->key);
maybe_push_operr(mres, res, resp->rc, 0);
}
}

if (resp->njson) {
pycbc_dict_add_text_kv(resultdict, "health_json", resp->json);
}
if (resp->rflags & LCB_RESP_F_FINAL) {
/* Note this can happen in both success and error cases!*/
operation_completed_with_err_info(parent, mres, cbtype, resp_base);
}

CB_THR_BEGIN(parent);
}



void
pycbc_callbacks_init(lcb_t instance)
{
Expand All @@ -813,6 +849,7 @@ pycbc_callbacks_init(lcb_t instance)
lcb_install_callback3(instance, LCB_CALLBACK_OBSERVE, observe_callback);
lcb_install_callback3(instance, LCB_CALLBACK_STATS, stats_callback);
lcb_install_callback3(instance, LCB_CALLBACK_PING, ping_callback);
lcb_install_callback3(instance, LCB_CALLBACK_DIAG, diag_callback);

/* Subdoc */
lcb_install_callback3(instance, LCB_CALLBACK_SDLOOKUP, subdoc_callback);
Expand Down

0 comments on commit 520ea27

Please sign in to comment.