From 6fd9ad02c4f5e9b3a139511ced1e7042f3e964bc Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Pereira Martins Date: Tue, 24 Jul 2018 21:49:37 +0200 Subject: [PATCH 1/3] formatting --- JumpScale9RecordChain/data/bcdb/BCDBTable.py | 127 +++++++++--------- JumpScale9RecordChain/data/schema/Schema.py | 52 +++---- .../data/schema/SchemaProperty.py | 26 ++-- 3 files changed, 100 insertions(+), 105 deletions(-) diff --git a/JumpScale9RecordChain/data/bcdb/BCDBTable.py b/JumpScale9RecordChain/data/bcdb/BCDBTable.py index f5a7bfe..e2fec28 100644 --- a/JumpScale9RecordChain/data/bcdb/BCDBTable.py +++ b/JumpScale9RecordChain/data/bcdb/BCDBTable.py @@ -6,10 +6,9 @@ JSBASE = j.application.jsbase_get_class() - class BCDBTable(JSBASE): - def __init__(self,bcdb,schema,name=""): + def __init__(self, bcdb, schema, name=""): """ """ @@ -26,42 +25,42 @@ def __init__(self,bcdb,schema,name=""): else: raise RuntimeError("input needs to be schema in text or obj format") - if self.name=="": + if self.name == "": self.name = schema.name if not schema.name: raise RuntimeError("schema name cannot be empty") - self.db = self.bcdb.zdbclient.namespace_new(name=name, maxsize=0, die=False) - self.index = j.core.db #only for now, will need better solution in future + self.db = self.bcdb.zdbclient.namespace_new(name=name, maxsize=0, die=False) + self.index = j.core.db # only for now, will need better solution in future self.schema = schema - self._index_key = "index:%s:%s"%(self.bcdb.zdbclient.instance,self.name) + self._index_key = "index:%s:%s" % (self.bcdb.zdbclient.instance, self.name) self.index_load() def index_delete(self): - for key in self.index.keys(self._index_key+"*"): + for key in self.index.keys(self._index_key + "*"): self.index.delete(key) def index_load(self): """ """ self.index_delete() - self.logger.debug("build index for: %s"%self.name) - def iload(id,data,result): + self.logger.debug("build index for: %s" % self.name) + + def iload(id, data, result): if data: - index,bdata = msgpack.unpackb(data) - self._index(index,id) + index, bdata = msgpack.unpackb(data) + self._index(index, id) self.db.iterate(iload) self.logger.debug("build index done") - def destroy(self): raise RuntimeError("not implemented yet, need to go to db and remove namespace") - def set(self,data,id=None,hook=None): + def set(self, data, id=None, hook=None): """ if string -> will consider to be json if binary -> will consider data for capnp @@ -69,9 +68,9 @@ def set(self,data,id=None,hook=None): if ddict will put inside JSOBJ @RETURN JSOBJ - + """ - if j.data.types.string.check(data): + if j.data.types.string.check(data): data = j.data.serializer.json.loads(data) obj = self.schema.get(data) elif j.data.types.bytes.check(data): @@ -79,90 +78,89 @@ def set(self,data,id=None,hook=None): elif "_JSOBJ" in data.__dict__: obj = data if id is None and obj.id is not None: - id=obj.id + id = obj.id elif j.data.types.dict.check(data): obj = self.schema.get(data) else: raise RuntimeError("Cannot find data type, str,bin,obj or ddict is only supported") bdata = obj.data - #we store data in list - l=[] - index={} + # we store data in list + l = [] + index = {} for item in self.schema.index_list: - r=eval("obj.%s"%item) + r = eval("obj.%s" % item) if r: - index[item]=r + index[item] = r - #later: + # later: acl = b"" crc = b"" signature = b"" - l=[index,bdata] + l = [index, bdata] data = msgpack.packb(l) if hook: - obj = hook(obj,index) + obj = hook(obj, index) - - if id==None: - #means a new one + if id == None: + # means a new one id = self.db.set(data) else: - id2 = self.db.set(data,id=id) + id2 = self.db.set(data, id=id) obj.id = id obj.index = index - self._index(index,id) - + self._index(index, id) + return obj - def _index(self,index,id): - - for key,item in index.items(): + def _index(self, index, id): + + for key, item in index.items(): if j.data.types.bytes.check(key): - key=key.decode() - key2=self._index_key+":%s"%key - res = self.index.hget(key2,item) - if res==None: - res = struct.pack(" Date: Wed, 25 Jul 2018 08:18:35 +0200 Subject: [PATCH 2/3] ZDBServer: change default addr to ip instead of host --- JumpScale9RecordChain/servers/zdb/ZDBServer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/JumpScale9RecordChain/servers/zdb/ZDBServer.py b/JumpScale9RecordChain/servers/zdb/ZDBServer.py index 54bc9df..6917c9b 100644 --- a/JumpScale9RecordChain/servers/zdb/ZDBServer.py +++ b/JumpScale9RecordChain/servers/zdb/ZDBServer.py @@ -4,7 +4,7 @@ import time TEMPLATE = """ -addr = "localhost" +addr = "127.0.0.1" port = 9900 path = "" mode = "" @@ -22,12 +22,12 @@ def __init__(self, instance, data={}, parent=None, interactive=False): """ JSConfigBase.__init__(self, instance=instance, data=data, parent=parent, template=TEMPLATE, ui=None, interactive=interactive) - + j.sal.fs.createDir(self.config.data["path"]) self._initdir() - def client_get(self, secrets="",encryptionkey=""): + def client_get(self, secrets="", encryptionkey=""): return j.clients.zdb.configure(instance=self.instance, secrets=secrets, adminsecret=self.config.data['adminsecret_'], @@ -52,9 +52,9 @@ def start(self): mode = self.config.data['mode'] - d=self.config.data - if j.sal.nettools.tcpPortConnectionTest(d["addr"],d["port"]): - r=j.clients.redis.get(ipaddr=d["addr"], port=d["port"]) + d = self.config.data + if j.sal.nettools.tcpPortConnectionTest(d["addr"], d["port"]): + r = j.clients.redis.get(ipaddr=d["addr"], port=d["port"]) r.ping() return() From fc71fd3eeed8ccce135d3fda07f6ca4f7587aa8e Mon Sep 17 00:00:00 2001 From: Christophe de Carvalho Pereira Martins Date: Wed, 25 Jul 2018 09:36:22 +0200 Subject: [PATCH 3/3] zdb: allow to delete data --- .../clients/zdb/ZDBClientNS.py | 116 ++++++++++-------- JumpScale9RecordChain/data/bcdb/BCDBTable.py | 9 +- 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/JumpScale9RecordChain/clients/zdb/ZDBClientNS.py b/JumpScale9RecordChain/clients/zdb/ZDBClientNS.py index b3cf5f8..358cf92 100644 --- a/JumpScale9RecordChain/clients/zdb/ZDBClientNS.py +++ b/JumpScale9RecordChain/clients/zdb/ZDBClientNS.py @@ -8,9 +8,10 @@ JSBASE = j.application.jsbase_get_class() + class ZDBClientNS(JSBASE): - def __init__(self,zdbclient,nsname ): + def __init__(self, zdbclient, nsname): """ is connection to ZDB @@ -27,26 +28,25 @@ def __init__(self,zdbclient,nsname ): self.zdbclient = zdbclient self.redis = j.clients.redis.get(ipaddr=zdbclient.config.data['addr'], - port=zdbclient.config.data['port'], - fromcache=False) + port=zdbclient.config.data['port'], + fromcache=False) self.redis = self._patch_redis_client(self.redis) - self.nsname = nsname.lower().strip() self.mode = self.zdbclient.mode if self.adminsecret is not "": self.redis.execute_command("AUTH", self.adminsecret) - #put secret on namespace & select namespace + # put secret on namespace & select namespace if self.secret is "": self.redis.execute_command("SELECT", self.nsname) else: self.redis.execute_command("SELECT", self.nsname, self.secret) @property - def adminsecret(self): + def adminsecret(self): return self.zdbclient.adminsecret @property @@ -60,28 +60,30 @@ def _patch_redis_client(self, redis): # don't auto parse response for set, cause it's not 100% redis compatible # 0-db does return a key after in set del redis.response_callbacks['SET'] + # don't auto parse response for del, 0-db returns OK for success instead of the number of key deleted + del redis.response_callbacks['DEL'] return redis - def _key_get(self,key,set=True): - - if self.mode=="seq": + def _key_get(self, key, set=True): + + if self.mode == "seq": if key is None: - key="" + key = "" else: - key = struct.pack("