From db434a2ecff6f2a450b4e68769fcfd47af50e48f Mon Sep 17 00:00:00 2001 From: jievince Date: Tue, 4 Aug 2020 12:49:04 +0800 Subject: [PATCH 01/17] add thrift wrapper for meta and storage --- GetFullGraph.py | 102 ++++++ nebula/ngData/data.py | 161 ++++++++++ nebula/ngMeta/MetaClient.py | 283 ++++++++++++++++ nebula/ngMeta/__init__.py | 0 nebula/ngStorage/StorageClient.py | 301 ++++++++++++++++++ nebula/ngStorage/__init__.py | 0 .../ngProcessor/ScanEdgeProcessor.py | 42 +++ .../ngProcessor/ScanVertexProcessor.py | 44 +++ 8 files changed, 933 insertions(+) create mode 100644 GetFullGraph.py create mode 100644 nebula/ngData/data.py create mode 100644 nebula/ngMeta/MetaClient.py create mode 100644 nebula/ngMeta/__init__.py create mode 100644 nebula/ngStorage/StorageClient.py create mode 100644 nebula/ngStorage/__init__.py create mode 100644 nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py create mode 100644 nebula/ngStorage/ngProcessor/ScanVertexProcessor.py diff --git a/GetFullGraph.py b/GetFullGraph.py new file mode 100644 index 00000000..9c4ff2da --- /dev/null +++ b/GetFullGraph.py @@ -0,0 +1,102 @@ +import sys, getopt +from nebula.ngMeta.MetaClient import MetaClient +from nebula.ngStorage.StorageClient import StorageClient +from nebula.ngStorage.ngProcessor.ScanEdgeProcessor import ScanEdgeProcessor +from nebula.ngStorage.ngProcessor.ScanVertexProcessor import ScanVertexProcessor + + +def scanEdge(space, returnCols, allCols): + scanEdgeResponseIter = storageClient.scanEdge(space, returnCols, allCols, 100, 0, sys.maxsize) + scanEdgeResponse = scanEdgeResponseIter.next() + if scanEdgeResponse is None: + print('not ok') + else: + print('ok') + processEdge(space, scanEdgeResponse) + while scanEdgeResponseIter.hasNext(): + scanEdgeResponse = scanEdgeResponseIter.next() + if scanEdgeResponse is None: + print("Error occurs while scaning edge") + break + processEdge(space, scanEdgeResponse) + +def scanVertex(space, returnCols, allCols): + scanVertexResponseIter = storageClient.scanVertex(space, returnCols, allCols, 100, 0, sys.maxsize) + scanVertexResponse = scanVertexResponseIter.next() + if scanVertexResponse is None: + print('not ok') + else: + print('ok') + processVertex(space, scanVertexResponse) + while scanVertexResponseIter.hasNext(): + scanVertexResponse = scanVertexResponseIter.next() + if scanVertexResponse is None: + print("Error occurs while scaning vertex") + break + processVertex(space, scanVertexResponse) + +def processEdge(space, scanEdgeResponse): + result = scanEdgeProcessor.process(space, scanEdgeResponse) + # Get the corresponding rows by edgeName + for edgeName, edgeRows in result.rows.items(): + print('edgeName: ', edgeName) + for row in edgeRows: + srcId = row.defaultProperties[0].getValue() + dstId = row.defaultProperties[2].getValue() + print('srcId: ', srcId, ' dstId: ', dstId) + props = {} + for prop in row.properties: + propName = prop.getName() + propValue = prop.getValue() + props[propName] = propValue + print(props) + +def processVertex(space, scanVertexResponse): + result = scanVertexProcessor.process(space, scanVertexResponse) + if result is None: + print('processVertex: result is None') + return None + for tagName, tagRows in result.rows.items(): + print('tagName: ', tagName) + for row in tagRows: + vid = row.defaultProperties[0].getValue() + print('vid: ', vid) + props = {} + for prop in row.properties: + propName = prop.getName() + propValue = prop.getValue() + props[propName] = propValue + print(props) + +def getReturnCols(space): + tagItems = metaClient.getTags(space) + vertexReturnCols = {} + for tagItem in tagItems: + tagName = tagItem.tag_name + vertexReturnCols[tagName] = metaClient.getTagSchema(space, tagName).keys() + edgeItems = metaClient.getEdges(space) + edgeReturnCols = {} + for edgeItem in edgeItems: + edgeName = edgeItem.edge_name + edgeReturnCols[edgeName] = metaClient.getEdgeSchema(space, edgeName).keys() + + return vertexReturnCols, edgeReturnCols + + +if __name__ == '__main__': + metaClient = MetaClient([(sys.argv[1], sys.argv[2])]) + metaClient.connect() + storageClient = StorageClient(metaClient) + scanEdgeProcessor = ScanEdgeProcessor(metaClient) + scanVertexProcessor = ScanVertexProcessor(metaClient) + + spaceToRead = sys.argv[3] + vertexReturnCols, edgeReturnCols = getReturnCols(spaceToRead) + allCols = True + + if spaceToRead not in metaClient.getPartsAllocFromCache().keys(): + raise Exception('spaceToRead %s is not found in nebula') + else: + print('scaning space %s' % spaceToRead) + scanVertex(spaceToRead, vertexReturnCols, allCols) + scanEdge(spaceToRead, edgeReturnCols, allCols) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py new file mode 100644 index 00000000..83d7b699 --- /dev/null +++ b/nebula/ngData/data.py @@ -0,0 +1,161 @@ +from enum import Enum +import struct + +class PropertyDef: + PropertyType = Enum('PropertyType', ('UNKNOWN', 'BOOL', 'INT', 'VID', 'FLOAT', 'DOUBLE', \ + 'STRING', 'VERTEX_ID', 'TAG_ID', 'SRC_ID', 'EDGE_TYPE', 'EDGE_RANK', 'DST_ID')) + + def __init__(self, propertyType, name): + self.propertyType = propertyType + self.name = name + + +class Property: + def __init__(self, propertyType, name, value): + self.propertyDef = PropertyDef(propertyType, name) + self.value = value + + def getType(self): + return self.propertyDef.type + + def getName(self): + return self.propertyDef.name + + def getValue(self): + return self.value + +class Row: + def __init__(self, defaultProperties, properties): + self.defaultProperties = defaultProperties + self.properties = properties + + +class RowReader: + def __init__(self, schema, schemaVersion=0): + self.schemaVersion = schemaVersion + self.defs = [] + self.fieldNum = 0 + self.propertyNameIndex = {} + self.propertyTypes = [] + self.offset = 0 + + idx = 0 + for columnDef in schema.columns: + propertyType = PropertyDef.PropertyType(columnDef.type.type+1) # ColumnDef is in common/ttypes.py + columnName = columnDef.name + self.defs.append((columnName, propertyType)) + self.propertyNameIndex[columnName] = idx + idx += 1 + self.fieldNum = len(self.defs) + + def decodeValue(self, value, schemaVersion=None): + if schemaVersion is None: + schemaVersion = self.schemaVersion + self.offset = 0 + blkOffBytesNum = value[0] & 0x07 + 1 + verBytesNum = value[0] >> 5 + self.offset += 1 + ver = 0 + if verBytesNum > 0: + for i in range(verBytesNum): + ver |= value[self.offset] << 8 + self.offSet += 1 + # print('blkOffBytesNum: ', blkOffBytesNum, ' verBytesNum: ', verBytesNum, ' ver: ', ver, ' schemaVersion: ', schemaVersion) + if ver != schemaVersion: + raise Exception('parsed version %d is not equal to version %d provided', ver, schemaVersion) + self.offset += blkOffBytesNum * (self.fieldNum // 16) + properties = [] + for i in range(len(self.defs)): + field = self.defs[i][0] + propertyType = self.defs[i][1] + if propertyType == PropertyDef.PropertyType.BOOL: + properties.append(self.getBoolProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.INT: + properties.append(self.getIntProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.FLOAT: #unused now + properties.append(self.getFloatProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.DOUBLE: + properties.append(self.getDoubleProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.STRING: + properties.append(self.getStringProperty(field, value)) + else: + raise Exception('Invalid propertyType in schema: ', propertyType) + + return properties + + def edgeKey(self, srcId, edgeType, dstId): + properties = [] + properties.append(Property(PropertyDef.PropertyType.SRC_ID, "_srcId", srcId)) + properties.append(Property(PropertyDef.PropertyType.EDGE_TYPE, "_edgeType", edgeType)) + properties.append(Property(PropertyDef.PropertyType.DST_ID, "_dstId", dstId)) + return properties + + def vertexKey(self, vertexId, tagId): + properties = [] + properties.append(Property(PropertyDef.PropertyType.VERTEX_ID, "_vertexId", vertexId)) + properties.append(Property(PropertyDef.PropertyType.TAG_ID, "_tagId", tagId)) + return properties + + def getProperty(self, row, name): + if name not in propertyNameIndex.keys(): + return None + return row.properties[propertyNameIndex[name]] + + def getPropertyByIndex(self, row, index): + if index < 0 or index >= len(row.getProperties()): + return None + return row.properties[index] + + def getBoolProperty(self, name, value): + val = value[self.offset] != 0x00 + self.offset += 1 + return Property(PropertyDef.PropertyType.BOOL, name, val) + + def getIntProperty(self, name, value): + val = self.readCompressedInt(value) + return Property(PropertyDef.PropertyType.INT, name, val) #### 字节流解析出data + + def getFloatProperty(self, name, value): + val = struct.unpack_from('= 0: + break + val |= (byteV & 0x7f) << shift + curOff += 1 + shift += 7 + if curOff == len(value): + return None + val |= byteV << shift + curOff += 1 + #print('readCompressedInt: ', value[self.offset:curOff], 'val is: ', val) + self.offset = curOff + return val + + +class Result: + def __init__(self, rows): + self.rows = rows + self.size = 0 + for entry in rows: + self.size += len(rows[entry]) diff --git a/nebula/ngMeta/MetaClient.py b/nebula/ngMeta/MetaClient.py new file mode 100644 index 00000000..390ddc2a --- /dev/null +++ b/nebula/ngMeta/MetaClient.py @@ -0,0 +1,283 @@ +import random +import socket +import struct + +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol + +from meta.MetaService import Client +from meta.ttypes import EdgeItem +from meta.ttypes import ErrorCode +from meta.ttypes import GetEdgeReq +from meta.ttypes import GetEdgeResp +from meta.ttypes import GetPartsAllocReq +from meta.ttypes import GetPartsAllocResp +from meta.ttypes import GetTagReq +from meta.ttypes import GetTagResp +from meta.ttypes import ListHostsReq +from meta.ttypes import ListHostsResp +from meta.ttypes import ListEdgesReq +from meta.ttypes import ListEdgesResp +from meta.ttypes import ListSpacesReq +from meta.ttypes import ListSpacesResp +from meta.ttypes import ListTagsReq +from meta.ttypes import ListTagsResp +from meta.ttypes import TagItem + +class MetaClient: + def __init__(self, addresses, timeout=1000, + connectionRetry=3, executionRetry=3): + self.addresses = addresses + self.timeout = timeout + self.connectionRetry = connectionRetry + self.executionRetry = executionRetry + self.spaceNameMap = {} # map + self.spacePartLocation = {} # map>> + self.spacePartLeader = {} # map> + self.spaceTagItems = {} # map> + self.spaceEdgeItems = {} # map + self.tagNameMap = {} # map + self.edgeNameMap = {} + self.metaClient = None + + def connect(self): + while self.connectionRetry > 0: + code = self.doConnect(self.addresses) + if code == 0: + return ErrorCode.SUCCEEDED + self.connectionRetry -= 1 + return ErrorCode.E_FAIL_TO_CONNECT + + def doConnect(self, addresses): + address = addresses[random.randint(0, len(addresses)-1)] + #print('metaClient is connecting to: ', address[0], address[1]) + host = address[0] + port = address[1] + tTransport = TSocket.TSocket(host, port) + tTransport.setTimeout(self.timeout) + tTransport = TTransport.TBufferedTransport(tTransport) + tProtocol = TBinaryProtocol.TBinaryProtocol(tTransport) + tTransport.open() + self.metaClient = Client(tProtocol) + + for spaceIdName in self.listSpaces(): + spaceName = spaceIdName.name # class IdName + self.spaceNameMap[spaceName] = spaceIdName.id.get_space_id() + #print(spaceName, spaceIdName.id.get_space_id()) + self.spacePartLocation[spaceName] = self.getPartsAlloc(spaceName) + self.spacePartLeader[spaceName] = {} + # Loading tag schema's cache + tags = {} + tagsName = {} + for tagItem in self.getTags(spaceName): + tags[tagItem.tag_name] = tagItem + tagsName[tagItem.tag_id] = tagItem.tag_name + + self.spaceTagItems[spaceName] = tags + self.tagNameMap[spaceName] = tagsName + + # Loading edge schema's cache + edges = {} + edgesName = {} + for edgeItem in self.getEdges(spaceName): + edges[edgeItem.edge_name] = edgeItem + edgesName[edgeItem.edge_type] = edgeItem.edge_name + self.spaceEdgeItems[spaceName] = edges + self.edgeNameMap[spaceName] = edgesName + # Get part leader + self.getSpacePartLeader() + + return 0 + + def getSpaceIdFromCache(self, spaceName): + if spaceName not in self.spaceNameMap.keys(): + return -1 + else: + return self.spaceNameMap[spaceName] + + def getSpacePartLeaderFromCache(self, spaceName, partId): + if spaceName not in self.spacePartLeader.keys(): + return None + if partId not in self.spacePartLeader[spaceName].keys(): + return None + return self.spacePartLeader[spaceName][partId] + + def updateSpacePartLeader(self, spaceName, partId, leader): + self.spacePartLeader[spaceName][partId] = leader + + def getSpacePartLeader(self): + listHostsReq = ListHostsReq() + listHostsResp = self.metaClient.listHosts(listHostsReq) + if listHostsResp.code != ErrorCode.SUCCEEDED: + print('getSpacePartLeader error, eror code: ', listHostsResp.code) + return None + + for hostItem in listHostsResp.hosts: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostItem.hostAddr.ip & 0xffffffff))) + port = hostItem.hostAddr.port + leader = (host, port) + for space, partIds in hostItem.leader_parts.items(): + for partId in partIds: + self.spacePartLeader[space][partId] = leader + #print('getSpacePartLeader: ', self.spacePartLeader) + + def listSpaces(self): + listSpacesReq = ListSpacesReq() + listSpacesResp = self.metaClient.listSpaces(listSpacesReq) + if listSpacesResp.code == ErrorCode.SUCCEEDED: + return listSpacesResp.spaces########## spaceNameID--> IdName + else: + print('list spaces error, error code: ', listSpacesResp.code) + return None + + def getPartFromCache(self, spaceName, part): + if spaceName not in self.spacePartLocation.keys(): + return None + partsAlloc = self.spacePartLocation[spaceName] + if partsAlloc is None or part not in partsAlloc.keys(): + return None + return partsAlloc[part] + + def getPartsAlloc(self, spaceName): + spaceId = self.getSpaceIdFromCache(spaceName) + if spaceId == -1: + return None + getPartsAllocReq = GetPartsAllocReq(spaceId) + getPartsAllocResp = self.metaClient.getPartsAlloc(getPartsAllocReq) + + if getPartsAllocResp.code == ErrorCode.SUCCEEDED: + addressMap = {} + for partId, hostAddrs in getPartsAllocResp.parts.items(): + addresses = [] + for hostAddr in hostAddrs: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostAddr.ip & 0xffffffff))) + port = hostAddr.port + addresses.append((host, port)) + addressMap[partId] = addresses + + return addressMap + else: + print("get parts alloc error, error code: ", getPartsAllocResp.code) + return None + + def getPartsAllocFromCache(self): + return self.spacePartLocation + + def getPartAllocFromCache(self, spaceName, part): + if spaceName in self.spacePartLocation.keys(): + partsAlloc = self.spacePartLocation[spaceName] + if part in partsAlloc.keys(): + return partsAlloc[part] + + return None + + def getTagItemFromCache(self, spaceName, tagName): + if spaceName not in self.spaceTagItems.keys(): + tags = {} + for tagItem in self.getTags(spaceName): + tags[tagItem.tag_name] = tagItem + self.spaceTagItems[spaceName] = tags + + tagItems = self.spaceTagItems[spaceName] + if tagName in tagItems.keys(): + return tagItems[tagName] + return None + + def getTagNameFromCache(self, spaceName, tagId): + if spaceName in self.tagNameMap.keys(): + tagNames = self.tagNameMap[spaceName] + if tagId in tagNames.keys(): + return tagNames[tagId] + + return None + + def getTags(self, spaceName): + spaceId = self.getSpaceIdFromCache(spaceName) + if spaceId == -1: + return None + listTagsReq = ListTagsReq(spaceId) + listTagsResp = self.metaClient.listTags(listTagsReq) + + if listTagsResp.code == ErrorCode.SUCCEEDED: + return listTagsResp.tags + else: + print('get tags error, error code: ', listTagsResp.code) + return None + + def getTag(self, spaceName, tagName): + spaceId = self.getSpaceIdFromCache(spaceName) + getTagReq = GetTagReq(spaceId, tagName, -1) + getTagResp = self.metaClient.getTag(getTagReq) + + if getTagResp.code == ErrorCode.SUCCEEDED: + return getTagResp.schema + else: + return None + + def getTagSchema(self, spaceName, tagName, version=-1): + spaceId = self.getSpaceIdFromCache(spaceName) + getTagReq = GetTagReq(spaceId, tagName, version) + getTagResp = self.metaClient.getTag(getTagReq) + result = {} + for columnDef in getTagResp.schema.columns: + result[columnDef.name] = columnDef.type.type # 需要将type convert to Class + + return result + + def getEdgeItemFromCache(self, spaceName, edgeName): + if spaceName not in self.spaceEdgeItems.keys(): + edges = {} + for edgeItem in self.getEdges(spaceName): + edges[edgeItem.edge_name] = edgeItem + self.spaceEdgeItems[spaceName] = edges + + edgeItems = self.spaceEdgeItems[spaceName] + if edgeName in edgeItems.keys(): + return edgeItems[edgeName] + else: + return None + + def getEdgeNameFromCache(self, spaceName, edgeType): + if spaceName in self.edgeNameMap.keys(): + edgeNames = self.edgeNameMap[spaceName] + if edgeType in edgeNames.keys(): + return edgeNames[edgeType] + + return None + + def getEdges(self, spaceName): + spaceId = self.getSpaceIdFromCache(spaceName) + if spaceId == -1: + return None + listEdgesReq = ListEdgesReq(spaceId) + listEdgesResp =self.metaClient.listEdges(listEdgesReq) + if listEdgesResp.code == ErrorCode.SUCCEEDED: + return listEdgesResp.edges + else: + print('get tags error, error code: ', listEdgesResp.code) + return None + + def getEdge(self, spaceName, edgeName): + spaceId = self.getSpaceIdFromCache(spaceName) + if spaceId == -1: + return None + getEdgeReq = GetEdgeReq(spaceId, edgeName, -1) + getEdgeResp = self.metaClient.getEdge(getEdgeReq) + if getEdgeResp.code == ErrorCode.SUCCEEDED: + return getEdgeResp.Schema + else: + print('get edge error, error code: ', getEdgeResp.code) + return None + + def getEdgeSchema(self, spaceName, edgeName, version=-1): + spaceId = self.getSpaceIdFromCache(spaceName) + if spaceId == -1: + return None + getEdgeReq = GetEdgeReq(spaceId, edgeName, version) + getEdgeResp = self.metaClient.getEdge(getEdgeReq) + result = {} + for columnDef in getEdgeResp.schema.columns: + result[columnDef.name] = columnDef.type.type + + return result diff --git a/nebula/ngMeta/__init__.py b/nebula/ngMeta/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py new file mode 100644 index 00000000..2b7d2ee7 --- /dev/null +++ b/nebula/ngStorage/StorageClient.py @@ -0,0 +1,301 @@ +import socket +import struct +import random + +from thrift.transport import TTransport +from thrift.transport import TSocket +from thrift.protocol import TBinaryProtocol + +from storage.StorageService import Client +from storage.ttypes import EntryId +from storage.ttypes import PropDef +from storage.ttypes import PropOwner +from storage.ttypes import ScanEdgeRequest +from storage.ttypes import ScanVertexRequest +from storage.ttypes import ErrorCode + +class Iterator: + def __init__(self, it=None): + self.it = iter(it) + self._hasnext = None + + def __iter__(self): + return self + + def next(self): + if self._hasnext: + result = self._thenext + else: + result = next(self.it) + self._hasnext = None + return result + + def hasNext(self): + if self._hasnext is None: + try: self._thenext = next(self.it) + except StopIteration: self._hasnext = False + else: self._hasnext = True + return self._hasnext + + +class ScanEdgeResponseIter: + def __init__(self, clientDad, space, leader, scanEdgeRequest, client): + self.clientDad = clientDad + self.space = space + self.leader = leader + self.scanEdgeRequest = scanEdgeRequest + self.client = client + self.cursor = None + self.haveNext = True + + def hasNext(self): + return self.haveNext + + def next(self): + self.scanEdgeRequest.cursor = self.cursor + scanEdgeResponse = self.client.scanEdge(self.scanEdgeRequest) + assert(scanEdgeResponse is not None), 'scanEdgeReponse is none' + self.cursor = scanEdgeResponse.next_cursor + self.haveNext = scanEdgeResponse.has_next + + if not self.clientDad.isSuccessfully(scanEdgeResponse): + print('scanEdgeResponse is not successfully, failed_codes: ', scanEdgeResponse.result.failed_codes) + self.leader, self.client = self.clientDad.handleResultCodes(scanEdgeResponse.result.failed_codes, self.space) + self.haveNext = False + return None + else: + return scanEdgeResponse + + return None + + +class ScanVertexResponseIter: + def __init__(self, clientDad, space, leader, scanVertexRequest, client): + self.clientDad = clientDad + self.space = space + self.leader = leader + self.scanVertexRequest = scanVertexRequest + self.client = client + self.cursor = None + self.haveNext = True + + def hasNext(self): + return self.haveNext + + def next(self): + self.scanVertexRequest.cursor = self.cursor + scanVertexResponse = self.client.scanVertex(self.scanVertexRequest) + assert(scanVertexResponse is not None), 'scanVertexReponse is none' + self.cursor = scanVertexResponse.next_cursor + self.haveNext = scanVertexResponse.has_next + + if not self.clientDad.isSuccessfully(scanVertexResponse): + print('scanVertexResponse is not successfully, failed_codes: ', scanVertexResponse.result.failed_codes) + self.leader, self.client = self.clientDad.handleResultCodes(scanVertexResponse.result.failed_codes, self.space) + self.haveNext = False + return None + else: + return scanVertexResponse + + return None + + +class ScanSpaceEdgeResponseIter: + def __init__(self, clientDad, space, partIdsIter, returnCols, allCols, limit, startTime, endTime): + self.clientDad = clientDad + self.scanEdgeResponseIter = None + self.space = space + self.partIdsIter = partIdsIter + self.returnCols = returnCols + self.allCols = allCols + self.limit = limit + self.startTime = startTime + self.endTime = endTime + + def hasNext(self): + return self.partIdsIter.hasNext() or self.scanEdgeResponseIter.hasNext() + + def next(self): + if self.scanEdgeResponseIter is None or not self.scanEdgeResponseIter.hasNext(): + part = self.partIdsIter.next() # 判断part.hasNext()??? is None? + leader = self.clientDad.getLeader(self.space, part) + if leader is None: + raise Exception('part %s not found in space %s' % (part, self.space)) + spaceId = self.clientDad.metaClient.getSpaceIdFromCache(self.space) + if spaceId == -1: + raise Exception('space %s not found' % self.space) + colums = self.clientDad.getEdgeReturnCols(self.space, self.returnCols) + scanEdgeRequest = ScanEdgeRequest(spaceId, part, None, colums, self.allCols, self.limit, self.startTime, self.endTime) + self.scanEdgeResponseIter = self.clientDad.doScanEdge(self.space, leader, scanEdgeRequest) + assert(self.scanEdgeResponseIter is not None), 'scanEdgeResponseIter is None' + + return self.scanEdgeResponseIter.next() + + +class ScanSpaceVertexResponseIter: + def __init__(self, clientDad, space, partIdsIter, returnCols, allCols, limit, startTime, endTime): + self.clientDad = clientDad + self.scanVertexResponseIter = None + self.space = space + self.partIdsIter = partIdsIter + self.returnCols = returnCols + self.allCols = allCols + self.limit = limit + self.startTime = startTime + self.endTime = endTime + + def hasNext(self): + return self.partIdsIter.hasNext() or self.scanVertexResponseIter.hasNext() + + def next(self): + if self.scanVertexResponseIter is None or not self.scanVertexResponseIter.hasNext(): + part = self.partIdsIter.next() # 判断part.hasNext()??? is None? + leader = self.clientDad.getLeader(self.space, part) + if leader is None: + raise Exception('part %s not found in space %s' % (part, self.space)) + spaceId = self.clientDad.metaClient.getSpaceIdFromCache(self.space) + if spaceId == -1: + raise Exception('space %s not found' % self.space) + colums = self.clientDad.getVertexReturnCols(self.space, self.returnCols) + scanVertexRequest = ScanVertexRequest(spaceId, part, None, colums, self.allCols, self.limit, self.startTime, self.endTime) + self.scanVertexResponseIter = self.clientDad.doScanVertex(self.space, leader, scanVertexRequest) + assert(self.scanVertexResponseIter is not None), 'scanVertexResponseIter is None' + + return self.scanVertexResponseIter.next() + + +class StorageClient: + def __init__(self, metaClient): + self.metaClient = metaClient + self.clients = {} + self.leaders = {} + self.timeout = 1000 + + def connect(self, address): + if address not in self.clients.keys(): + client = self.doConnect(address) + self.clients[address] = client + return client + else: + return self.clients[address] + + def disConnect(self, address): + self.clients.remove(address) + + def doConnects(self, addresses): + for address in addresses: + client = self.doConnect(address) + self.clients[address] = client + + def doConnect(self, address): + host = address[0] + port = address[1] + #print('StorageClient is connect to: tTransport ip: %s, port: %s' % (host, port)) + tTransport = TSocket.TSocket(host, port) + tTransport.setTimeout(self.timeout) + tTransport = TTransport.TBufferedTransport(tTransport) + tProtocol = TBinaryProtocol.TBinaryProtocol(tTransport) + tTransport.open() + return Client(tProtocol) + + def scanEdge(self, space, returnCols, allCols, limit, startTime, endTime): + partIds = self.metaClient.getPartsAllocFromCache()[space].keys() + partIdsIter = Iterator(partIds) + return ScanSpaceEdgeResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) + + def scanPartEdge(self, space, part, returnCols, allCols, limit, startTime, endTime): + spaceId = self.metaClient.getSpaceIdFromCache(space) + columns = self.getEdgeReturnCols(space, returnCols) + scanEdgeRequest = ScanEdgeRequest(spaceId, part, None, columns, allCols, limit, startTime, endTime) + leader = self.getLeader(space, part) + if leader is None: + raise Exception('part %s not found in space %s' % (part, space)) + return self.doScanEdge(space, leader, scanEdgeRequest) + + def scanVertex(self, space, returnCols, allCols, limit, startTime, endTime): + partIds = self.metaClient.getPartsAllocFromCache()[space].keys() + partIdsIter = Iterator(partIds) + return ScanSpaceVertexResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) + + def scanPartVertex(self, space, part, returnCols, allCols, limit, startTime, endTime): + spaceId = self.metaClient.getSpaceIdFromCache(space) + columns = self.getVertexReturnCols(space, returnCols) + scanVertexRequest = ScanVertexRequest(spaceId, part, None, columns, allCols, limit, startTime, endTie) + if leader is None: + raise Exception('part %s not found in space %s' % (part, space)) + return self.doScanVertex(space, leader, scanVertexRequest) + + def doScanEdge(self, space, leader, scanEdgeRequest): + client = self.connect(leader) + if client is None: + print('cannot connect to leader:', leader) + self.disConnect(leader) + return None + + return ScanEdgeResponseIter(self, space, leader, scanEdgeRequest, client) + + def doScanVertex(self, space, leader, scanVertexRequest): + client = self.connect(leader) + if client is None: + print('cannot connect to leader:', leader) + self.disConnect(leader) + return None + + return ScanVertexResponseIter(self, space, leader, scanVertexRequest, client) + + def getEdgeReturnCols(self, space, returnCols): + columns = {} + for edgeName, propNames in returnCols.items(): + edgeItem = self.metaClient.getEdgeItemFromCache(space, edgeName) + if edgeItem is None: + raise Exception('edge %s not found in space %s' % (edgeName, space)) + edgeType = edgeItem.edge_type + entryId = EntryId(edge_type=edgeType) + propDefs = [] + for propName in propNames: + propDef = PropDef(PropOwner.EDGE, entryId, propName) + propDefs.append(propDef) + columns[edgeType] = propDefs + return columns + + def getVertexReturnCols(self, space, returnCols): + columns = {} + for tagName, propNames in returnCols.items(): + tagItem = self.metaClient.getTagItemFromCache(space, tagName) + if tagItem is None: + raise Exception('tag %s not found in space %s' % (tagName, space)) + tagId = tagItem.tag_id + entryId = EntryId(tag_id=tagId) + propDefs = [] + for propName in propNames: + propDef = PropDef(PropOwner.SOURCE, entryId, propName) + propDefs.append(propDef) + columns[tagId] = propDefs + return columns + + def getLeader(self, spaceName, part): + return self.metaClient.getSpacePartLeaderFromCache(spaceName, part) + + def updateLeader(self, spaceName, partId, leader): + self.metaClient.updateSpacePartLeader(spaceName, partId, leader) + + def handleResultCodes(self, failedCodes, space): + for resultCode in failedCodes: + if resultCode.code == ErrorCode.E_LEADER_CHANGED: + print('ErrorCode.E_LEADER_CHANGED, leader changed to :', resultCode.leader) + hostAddr = resultCode.leader + if hostAddr is not None and hostAddr.ip != 0 and hostAddr.port != 0: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostAddr.ip & 0xffffffff))) + port = hostAddr.port + newLeader = (host, port) + self.updateLeader(space, code.part_id, newLeader) + if newLeader in self.clients.keys(): + newClient = self.clients[newLeader] + else: + newClient = None + return newLeader, newClient + + return None, None + + def isSuccessfully(self, response): + return len(response.result.failed_codes) == 0 diff --git a/nebula/ngStorage/__init__.py b/nebula/ngStorage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py new file mode 100644 index 00000000..0f6b993d --- /dev/null +++ b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py @@ -0,0 +1,42 @@ +from meta.MetaService import Client +from storage.ttypes import ScanEdgeResponse + +from nebula.ngData.data import Row +from nebula.ngData.data import RowReader +from nebula.ngData.data import Result + +class ScanEdgeProcessor: + def __init__(self, metaClient): + self.metaClient = metaClient + + def process(self, spaceName, scanEdgeResponse): + rowReaders = {} + rows = {} + edgeTypeNameMap = {} + + if scanEdgeResponse.edge_schema is not None: + for edgeType, schema in scanEdgeResponse.edge_schema.items(): + edgeName = self.metaClient.getEdgeNameFromCache(spaceName, edgeType) + edgeItem = self.metaClient.getEdgeItemFromCache(spaceName, edgeName) + schemaVersion = edgeItem.version + rowReaders[edgeType] = RowReader(schema, schemaVersion) + rows[edgeName] = [] ### + edgeTypeNameMap[edgeType] = edgeName + else: + print('scanEdgeResponse.edge_schema is None') + + if scanEdgeResponse.edge_data is not None: + for scanEdge in scanEdgeResponse.edge_data: + edgeType = scanEdge.type + if edgeType not in rowReaders.keys(): + continue + + rowReader = rowReaders[edgeType] + defaultProperties = rowReader.edgeKey(scanEdge.src, scanEdge.type, scanEdge.dst) + properties = rowReader.decodeValue(scanEdge.value) + edgeName = edgeTypeNameMap[edgeType] + rows[edgeName].append(Row(defaultProperties, properties)) + else: + print('scanEdgeResponse.edge_data is None') + + return Result(rows) diff --git a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py new file mode 100644 index 00000000..352248f7 --- /dev/null +++ b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py @@ -0,0 +1,44 @@ +from meta.MetaService import Client +from storage.ttypes import ScanVertexResponse + +from nebula.ngData.data import Row +from nebula.ngData.data import RowReader +from nebula.ngData.data import Result + +class ScanVertexProcessor: + def __init__(self, metaClient): + self.metaClient = metaClient + + def process(self, spaceName, scanVertexResponse): + if scanVertexResponse is None: + print('process: scanVertexResponse is None') + return None + rowReaders = {} + rows = {} + tagIdNameMap = {} + if scanVertexResponse.vertex_schema is not None: + for tagId, schema in scanVertexResponse.vertex_schema.items(): + tagName = self.metaClient.getTagNameFromCache(spaceName, tagId) + tagItem = self.metaClient.getTagItemFromCache(spaceName, tagName) + schemaVersion = tagItem.version + rowReaders[tagId] = RowReader(schema, schemaVersion) + rows[tagName] = [] ### + tagIdNameMap[tagId] = tagName + else: + print('scanVertexResponse.vertex_schema is None') + + if scanVertexResponse.vertex_data is not None: + for scanTag in scanVertexResponse.vertex_data: + tagId = scanTag.tagId + if tagId not in rowReaders.keys(): + continue + + rowReader = rowReaders[tagId] + defaultProperties = rowReader.vertexKey(scanTag.vertexId, scanTag.tagId) + properties = rowReader.decodeValue(scanTag.value) + tagName = tagIdNameMap[tagId] + rows[tagName].append(Row(defaultProperties, properties)) + else: + print('scanVertexResponse.vertex_data is None') + + return Result(rows) From 108b7c57821fc18f16408cebf8586ed0232ecf7a Mon Sep 17 00:00:00 2001 From: jievince Date: Fri, 7 Aug 2020 11:43:20 +0800 Subject: [PATCH 02/17] add tests and add compatibility with python2 --- .../StorageClientExample.py | 14 +++ nebula/ngData/__init__.py | 0 nebula/ngData/data.py | 85 ++++++++----- nebula/ngMeta/MetaClient.py | 102 +++++++++------ nebula/ngStorage/StorageClient.py | 30 +++-- .../ngProcessor/ScanEdgeProcessor.py | 7 ++ .../ngProcessor/ScanVertexProcessor.py | 7 ++ nebula/ngStorage/ngProcessor/__init__.py | 0 tests/test_StorageClient.py | 119 ++++++++++++++++++ tests/test_client.py | 2 +- 10 files changed, 288 insertions(+), 78 deletions(-) rename GetFullGraph.py => examples/StorageClientExample.py (93%) create mode 100644 nebula/ngData/__init__.py create mode 100644 nebula/ngStorage/ngProcessor/__init__.py create mode 100644 tests/test_StorageClient.py diff --git a/GetFullGraph.py b/examples/StorageClientExample.py similarity index 93% rename from GetFullGraph.py rename to examples/StorageClientExample.py index 9c4ff2da..3f1a86ca 100644 --- a/GetFullGraph.py +++ b/examples/StorageClientExample.py @@ -1,4 +1,18 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + +""" +Nebula StorageClient example. +""" + import sys, getopt + +sys.path.insert(0, '../') + from nebula.ngMeta.MetaClient import MetaClient from nebula.ngStorage.StorageClient import StorageClient from nebula.ngStorage.ngProcessor.ScanEdgeProcessor import ScanEdgeProcessor diff --git a/nebula/ngData/__init__.py b/nebula/ngData/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 83d7b699..8f0cad78 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -1,5 +1,13 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + from enum import Enum import struct +import six class PropertyDef: PropertyType = Enum('PropertyType', ('UNKNOWN', 'BOOL', 'INT', 'VID', 'FLOAT', 'DOUBLE', \ @@ -49,39 +57,47 @@ def __init__(self, schema, schemaVersion=0): self.fieldNum = len(self.defs) def decodeValue(self, value, schemaVersion=None): - if schemaVersion is None: - schemaVersion = self.schemaVersion - self.offset = 0 + if schemaVersion is None: + schemaVersion = self.schemaVersion + self.offset = 0 + # need to check if value is valid + if six.PY2: + blkOffBytesNum = ord(value[0]) & 0x07 + 1 + verBytesNum = ord(value[0]) >> 5 + else: blkOffBytesNum = value[0] & 0x07 + 1 verBytesNum = value[0] >> 5 - self.offset += 1 - ver = 0 - if verBytesNum > 0: - for i in range(verBytesNum): - ver |= value[self.offset] << 8 - self.offSet += 1 - # print('blkOffBytesNum: ', blkOffBytesNum, ' verBytesNum: ', verBytesNum, ' ver: ', ver, ' schemaVersion: ', schemaVersion) - if ver != schemaVersion: - raise Exception('parsed version %d is not equal to version %d provided', ver, schemaVersion) - self.offset += blkOffBytesNum * (self.fieldNum // 16) - properties = [] - for i in range(len(self.defs)): - field = self.defs[i][0] - propertyType = self.defs[i][1] - if propertyType == PropertyDef.PropertyType.BOOL: - properties.append(self.getBoolProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.INT: - properties.append(self.getIntProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.FLOAT: #unused now - properties.append(self.getFloatProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.DOUBLE: - properties.append(self.getDoubleProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.STRING: - properties.append(self.getStringProperty(field, value)) + self.offset += 1 + ver = 0 + if verBytesNum > 0: + for i in range(verBytesNum): + if six.PY2: + ver |= ord(value[self.offset]) << 8 else: - raise Exception('Invalid propertyType in schema: ', propertyType) + ver |= value[self.offset] << 8 + self.offSet += 1 + # print('blkOffBytesNum: ', blkOffBytesNum, ' verBytesNum: ', verBytesNum, ' ver: ', ver, ' schemaVersion: ', schemaVersion) + if ver != schemaVersion: + raise Exception('parsed version %d is not equal to version %d provided', ver, schemaVersion) + self.offset += blkOffBytesNum * (self.fieldNum // 16) + properties = [] + for i in range(len(self.defs)): + field = self.defs[i][0] + propertyType = self.defs[i][1] + if propertyType == PropertyDef.PropertyType.BOOL: + properties.append(self.getBoolProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.INT: + properties.append(self.getIntProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.FLOAT: #unused now + properties.append(self.getFloatProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.DOUBLE: + properties.append(self.getDoubleProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.STRING: + properties.append(self.getStringProperty(field, value)) + else: + raise Exception('Invalid propertyType in schema: ', propertyType) - return properties + return properties def edgeKey(self, srcId, edgeType, dstId): properties = [] @@ -107,7 +123,10 @@ def getPropertyByIndex(self, row, index): return row.properties[index] def getBoolProperty(self, name, value): - val = value[self.offset] != 0x00 + if six.PY2: + val = ord(value[self.offset]) != 0x00 + else: + val = value[self.offset] != 0x00 self.offset += 1 return Property(PropertyDef.PropertyType.BOOL, name, val) @@ -127,7 +146,11 @@ def getDoubleProperty(self, name, value): def getStringProperty(self, name, value): strLen = self.readCompressedInt(value) - val = str(value[self.offset:self.offset+strLen], 'utf-8') + #val = value[self.offset:self.offset+strLen].decode(encoding='utf-8') + if six.PY2: + val = str(value[self.offset:self.offset+strLen]) + else: + val = str(value[self.offset:self.offset+strLen], encoding='utf-8') self.offset += strLen return Property(PropertyDef.PropertyType.STRING, name, val) diff --git a/nebula/ngMeta/MetaClient.py b/nebula/ngMeta/MetaClient.py index 390ddc2a..185d18b8 100644 --- a/nebula/ngMeta/MetaClient.py +++ b/nebula/ngMeta/MetaClient.py @@ -1,7 +1,15 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + import random import socket import struct - +import six +import threading from thrift.transport import TTransport from thrift.transport import TSocket from thrift.protocol import TBinaryProtocol @@ -25,6 +33,21 @@ from meta.ttypes import ListTagsResp from meta.ttypes import TagItem +if six.PY3: + Timer = threading.Timer +else: + Timer = threading._Timer + +class RepeatTimer(Timer): + def __init__(self, interval, function): + Timer.__init__(self, interval, function) + self.daemon = True # set the RepeatTimer thread as a daemon thread, so it can end when main thread ends + + def run(self): + while not self.finished.wait(self.interval): + #print('daemon: ', self.daemon) + self.function(*self.args, **self.kwargs) + class MetaClient: def __init__(self, addresses, timeout=1000, connectionRetry=3, executionRetry=3): @@ -36,11 +59,11 @@ def __init__(self, addresses, timeout=1000, self.spacePartLocation = {} # map>> self.spacePartLeader = {} # map> self.spaceTagItems = {} # map> - self.spaceEdgeItems = {} # map - self.tagNameMap = {} # map - self.edgeNameMap = {} - self.metaClient = None - + self.spaceEdgeItems = {} # map> + self.tagNameMap = {} # map> + self.edgeNameMap = {} # map> + self.client = None + def connect(self): while self.connectionRetry > 0: code = self.doConnect(self.addresses) @@ -59,8 +82,14 @@ def doConnect(self, addresses): tTransport = TTransport.TBufferedTransport(tTransport) tProtocol = TBinaryProtocol.TBinaryProtocol(tTransport) tTransport.open() - self.metaClient = Client(tProtocol) + self.client = Client(tProtocol) + self.updateSchemas() + RepeatTimer(2, self.updateSchemas).start() # call updatSchemas() every 2 seconds + + return 0 + def updateSchemas(self): + print('threading active_count: ', threading.active_count()) for spaceIdName in self.listSpaces(): spaceName = spaceIdName.name # class IdName self.spaceNameMap[spaceName] = spaceIdName.id.get_space_id() @@ -85,9 +114,7 @@ def doConnect(self, addresses): edgesName[edgeItem.edge_type] = edgeItem.edge_name self.spaceEdgeItems[spaceName] = edges self.edgeNameMap[spaceName] = edgesName - # Get part leader - self.getSpacePartLeader() - + self.setSpacePartLeader() return 0 def getSpaceIdFromCache(self, spaceName): @@ -106,11 +133,11 @@ def getSpacePartLeaderFromCache(self, spaceName, partId): def updateSpacePartLeader(self, spaceName, partId, leader): self.spacePartLeader[spaceName][partId] = leader - def getSpacePartLeader(self): + def setSpacePartLeader(self): listHostsReq = ListHostsReq() - listHostsResp = self.metaClient.listHosts(listHostsReq) + listHostsResp = self.client.listHosts(listHostsReq) if listHostsResp.code != ErrorCode.SUCCEEDED: - print('getSpacePartLeader error, eror code: ', listHostsResp.code) + print('setSpacePartLeader error, eror code: ', listHostsResp.code) return None for hostItem in listHostsResp.hosts: @@ -120,11 +147,10 @@ def getSpacePartLeader(self): for space, partIds in hostItem.leader_parts.items(): for partId in partIds: self.spacePartLeader[space][partId] = leader - #print('getSpacePartLeader: ', self.spacePartLeader) def listSpaces(self): listSpacesReq = ListSpacesReq() - listSpacesResp = self.metaClient.listSpaces(listSpacesReq) + listSpacesResp = self.client.listSpaces(listSpacesReq) if listSpacesResp.code == ErrorCode.SUCCEEDED: return listSpacesResp.spaces########## spaceNameID--> IdName else: @@ -133,7 +159,7 @@ def listSpaces(self): def getPartFromCache(self, spaceName, part): if spaceName not in self.spacePartLocation.keys(): - return None + self.spacePartLocation[spaceName] = self.getPartsAlloc(spaceName) partsAlloc = self.spacePartLocation[spaceName] if partsAlloc is None or part not in partsAlloc.keys(): return None @@ -144,7 +170,7 @@ def getPartsAlloc(self, spaceName): if spaceId == -1: return None getPartsAllocReq = GetPartsAllocReq(spaceId) - getPartsAllocResp = self.metaClient.getPartsAlloc(getPartsAllocReq) + getPartsAllocResp = self.client.getPartsAlloc(getPartsAllocReq) if getPartsAllocResp.code == ErrorCode.SUCCEEDED: addressMap = {} @@ -173,15 +199,9 @@ def getPartAllocFromCache(self, spaceName, part): return None def getTagItemFromCache(self, spaceName, tagName): - if spaceName not in self.spaceTagItems.keys(): - tags = {} - for tagItem in self.getTags(spaceName): - tags[tagItem.tag_name] = tagItem - self.spaceTagItems[spaceName] = tags - - tagItems = self.spaceTagItems[spaceName] - if tagName in tagItems.keys(): - return tagItems[tagName] + if spaceName in self.spaceTagItems.keys() and tagName in self.spaceTagItems[spaceName].keys(): + return self.spaceTagItems[spaceName][tagName] + return None def getTagNameFromCache(self, spaceName, tagId): @@ -197,7 +217,7 @@ def getTags(self, spaceName): if spaceId == -1: return None listTagsReq = ListTagsReq(spaceId) - listTagsResp = self.metaClient.listTags(listTagsReq) + listTagsResp = self.client.listTags(listTagsReq) if listTagsResp.code == ErrorCode.SUCCEEDED: return listTagsResp.tags @@ -205,10 +225,10 @@ def getTags(self, spaceName): print('get tags error, error code: ', listTagsResp.code) return None - def getTag(self, spaceName, tagName): + def getTag(self, spaceName, tagName, version=-1): spaceId = self.getSpaceIdFromCache(spaceName) - getTagReq = GetTagReq(spaceId, tagName, -1) - getTagResp = self.metaClient.getTag(getTagReq) + getTagReq = GetTagReq(spaceId, tagName, version) + getTagResp = self.client.getTag(getTagReq) if getTagResp.code == ErrorCode.SUCCEEDED: return getTagResp.schema @@ -217,12 +237,15 @@ def getTag(self, spaceName, tagName): def getTagSchema(self, spaceName, tagName, version=-1): spaceId = self.getSpaceIdFromCache(spaceName) + print('spaceId: ', spaceId) + if spaceId == -1: + return None getTagReq = GetTagReq(spaceId, tagName, version) - getTagResp = self.metaClient.getTag(getTagReq) + getTagResp = self.client.getTag(getTagReq) result = {} for columnDef in getTagResp.schema.columns: - result[columnDef.name] = columnDef.type.type # 需要将type convert to Class - + result[columnDef.name] = columnDef.type.type + print('result: ', result) return result def getEdgeItemFromCache(self, spaceName, edgeName): @@ -251,19 +274,19 @@ def getEdges(self, spaceName): if spaceId == -1: return None listEdgesReq = ListEdgesReq(spaceId) - listEdgesResp =self.metaClient.listEdges(listEdgesReq) + listEdgesResp =self.client.listEdges(listEdgesReq) if listEdgesResp.code == ErrorCode.SUCCEEDED: return listEdgesResp.edges else: print('get tags error, error code: ', listEdgesResp.code) return None - def getEdge(self, spaceName, edgeName): + def getEdge(self, spaceName, edgeName, version=-1): spaceId = self.getSpaceIdFromCache(spaceName) if spaceId == -1: return None - getEdgeReq = GetEdgeReq(spaceId, edgeName, -1) - getEdgeResp = self.metaClient.getEdge(getEdgeReq) + getEdgeReq = GetEdgeReq(spaceId, edgeName, version) + getEdgeResp = self.client.getEdge(getEdgeReq) if getEdgeResp.code == ErrorCode.SUCCEEDED: return getEdgeResp.Schema else: @@ -275,9 +298,10 @@ def getEdgeSchema(self, spaceName, edgeName, version=-1): if spaceId == -1: return None getEdgeReq = GetEdgeReq(spaceId, edgeName, version) - getEdgeResp = self.metaClient.getEdge(getEdgeReq) + getEdgeResp = self.client.getEdge(getEdgeReq) result = {} for columnDef in getEdgeResp.schema.columns: result[columnDef.name] = columnDef.type.type - + print('edge sapcId: ', spaceId) + print(result) return result diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index 2b7d2ee7..76b065cc 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -1,3 +1,10 @@ +# --coding:utf-8-- +# +# Copyright (c) 2020 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + import socket import struct import random @@ -32,9 +39,12 @@ def next(self): def hasNext(self): if self._hasnext is None: - try: self._thenext = next(self.it) - except StopIteration: self._hasnext = False - else: self._hasnext = True + try: + self._thenext = next(self.it) + except StopIteration: + self._hasnext = False + else: + self._hasnext = True return self._hasnext @@ -117,7 +127,9 @@ def hasNext(self): def next(self): if self.scanEdgeResponseIter is None or not self.scanEdgeResponseIter.hasNext(): - part = self.partIdsIter.next() # 判断part.hasNext()??? is None? + part = self.partIdsIter.next() + if part is None: + return None leader = self.clientDad.getLeader(self.space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, self.space)) @@ -149,7 +161,10 @@ def hasNext(self): def next(self): if self.scanVertexResponseIter is None or not self.scanVertexResponseIter.hasNext(): - part = self.partIdsIter.next() # 判断part.hasNext()??? is None? + part = self.partIdsIter.next() + if part is None: + return None + print('part: ', part) leader = self.clientDad.getLeader(self.space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, self.space)) @@ -220,7 +235,8 @@ def scanVertex(self, space, returnCols, allCols, limit, startTime, endTime): def scanPartVertex(self, space, part, returnCols, allCols, limit, startTime, endTime): spaceId = self.metaClient.getSpaceIdFromCache(space) columns = self.getVertexReturnCols(space, returnCols) - scanVertexRequest = ScanVertexRequest(spaceId, part, None, columns, allCols, limit, startTime, endTie) + scanVertexRequest = ScanVertexRequest(spaceId, part, None, columns, allCols, limit, startTime, endTime) + leader = self.getLeader(space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, space)) return self.doScanVertex(space, leader, scanVertexRequest) @@ -288,7 +304,7 @@ def handleResultCodes(self, failedCodes, space): host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostAddr.ip & 0xffffffff))) port = hostAddr.port newLeader = (host, port) - self.updateLeader(space, code.part_id, newLeader) + self.updateLeader(space, resultCode.part_id, newLeader) if newLeader in self.clients.keys(): newClient = self.clients[newLeader] else: diff --git a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py index 0f6b993d..3c6924ef 100644 --- a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py @@ -1,3 +1,10 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + from meta.MetaService import Client from storage.ttypes import ScanEdgeResponse diff --git a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py index 352248f7..e9413b95 100644 --- a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py @@ -1,3 +1,10 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + from meta.MetaService import Client from storage.ttypes import ScanVertexResponse diff --git a/nebula/ngStorage/ngProcessor/__init__.py b/nebula/ngStorage/ngProcessor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py new file mode 100644 index 00000000..7e336af8 --- /dev/null +++ b/tests/test_StorageClient.py @@ -0,0 +1,119 @@ +# --coding:utf-8-- +# +# Copyright (c) 2019 vesoft inc. All rights reserved. +# +# This source code is licensed under Apache 2.0 License, +# attached with Common Clause Condition 1.0, found in the LICENSES directory. + + +""" +Nebula StorageClient tests. +""" + +import pytest +import sys +import os +import time +import threading + +from storage.ttypes import EntryId +from storage.ttypes import PropDef +from storage.ttypes import PropOwner +from storage.ttypes import ResultCode +from storage.ttypes import ErrorCode +from common.ttypes import HostAddr + +sys.path.insert(0, '../') + +from graph import ttypes +from nebula.ConnectionPool import ConnectionPool +from nebula.Client import GraphClient +from nebula.Common import * +from nebula.ngStorage.StorageClient import StorageClient +from nebula.ngMeta.MetaClient import MetaClient + +def prepare(): + connection_pool = ConnectionPool(host, graph_port) + client = GraphClient(connection_pool) + if client.is_none(): + raise AuthException('Connect failed') + resp = client.authenticate('user', 'password') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('DROP SPACE IF EXISTS %s' % spaceName) + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE SPACE %s(partition_num=1)' % spaceName) + assert resp.error_code == 0, resp.error_msg + time.sleep(5) + resp = client.execute('USE %s' % spaceName) + assert resp.error_code == 0, resp.error_msg + time.sleep(5) + resp = client.execute('CREATE TAG player(name string, age int)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE EDGE follow(degree double)') + assert resp.error_code == 0, resp.error_msg + time.sleep(12) + resp = client.execute('INSERT VERTEX player(name, age) VALUES 1:(\'Bob\', 18)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX player(name, age) VALUES 2:(\'Tome\', 22)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT EDGE follow(degree) VALUES 1->2:(94.7)') + assert resp.error_code == 0, resp.error_msg + +def test_scanEdge(): + result = storageClient.scanEdge(spaceName, {'follow':['degree']}, True, 100, 0, sys.maxsize) + assert result is not None and result.next().edge_data is not None + +def test_scanVertex(): + result = storageClient.scanVertex(spaceName, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) + assert result is not None and result.next().vertex_data is not None + +def test_scanPartEdge(): + result = storageClient.scanPartEdge(spaceName, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) + assert result is not None and result.next().edge_data is not None + +def test_scanPartVertex(): + result = storageClient.scanPartVertex(spaceName, 1, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) + assert result is not None and result.next().vertex_data is not None + +def test_getTagSchema(): + result = metaClient.getTagSchema(spaceName, 'player') + expect = {'name':6, 'age':2} + assert result == expect + +def test_getEdgeSchema(): + result = metaClient.getEdgeSchema(spaceName, 'follow') + expect = {'degree': 5} + assert result == expect + +def test_getEdgeReturnCols(): + edgeItem = metaClient.getEdgeItemFromCache(spaceName, 'follow') + edgeType = edgeItem.edge_type + entryId = EntryId(edge_type=edgeType) + result = storageClient.getEdgeReturnCols(spaceName, {'follow':['degree']}) + expect = {edgeType:[PropDef(PropOwner.EDGE, entryId, 'degree')]} + assert result == expect + +def test_getVertexReturnCols(): + tagItem = metaClient.getTagItemFromCache(spaceName, 'player') + tagId = tagItem.tag_id + entryId = EntryId(tag_id=tagId) + result = storageClient.getVertexReturnCols(spaceName, {'player':['name', 'age']}) + expect = {tagId:[PropDef(PropOwner.SOURCE, entryId, 'name'), PropDef(PropOwner.SOURCE, entryId, 'age')]} + assert result == expect + +def test_handleResultCodes(): + failedCodes = [ResultCode(code=ErrorCode.E_LEADER_CHANGED, part_id=1, leader=HostAddr(ip=2130706433, port=storage_port))] + result, _ = storageClient.handleResultCodes(failedCodes, spaceName) + expect = (host, storage_port) + assert result == expect + + +host = '127.0.0.1' +meta_port = 45159 +graph_port = 36159 +storage_port = 44159 +spaceName = 'test_storage' +prepare() +metaClient = MetaClient([('127.0.0.1', meta_port)]) +metaClient.connect() +storageClient = StorageClient(metaClient) diff --git a/tests/test_client.py b/tests/test_client.py index 08aec83a..260a48cc 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -25,7 +25,7 @@ def get_port(): - return 3699 + return 36159 def create_pool(port): From ddaee0895fa1e7b28b7819fb79e795155eda342c Mon Sep 17 00:00:00 2001 From: jievince Date: Fri, 7 Aug 2020 11:46:14 +0800 Subject: [PATCH 03/17] fix port error --- tests/test_StorageClient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 7e336af8..2abaf4b2 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -109,9 +109,9 @@ def test_handleResultCodes(): host = '127.0.0.1' -meta_port = 45159 -graph_port = 36159 -storage_port = 44159 +meta_port = 45500 +graph_port = 3699 +storage_port = 44500 spaceName = 'test_storage' prepare() metaClient = MetaClient([('127.0.0.1', meta_port)]) From 4c615b0e184eb5e4620da414f57e611295d47918 Mon Sep 17 00:00:00 2001 From: jievince Date: Fri, 7 Aug 2020 12:00:31 +0800 Subject: [PATCH 04/17] fix import error --- tests/test_StorageClient.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 2abaf4b2..62c61c58 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -16,15 +16,13 @@ import time import threading +sys.path.insert(0, '../') from storage.ttypes import EntryId from storage.ttypes import PropDef from storage.ttypes import PropOwner from storage.ttypes import ResultCode from storage.ttypes import ErrorCode from common.ttypes import HostAddr - -sys.path.insert(0, '../') - from graph import ttypes from nebula.ConnectionPool import ConnectionPool from nebula.Client import GraphClient From b025cbf8ee226d3e552f9d2e13f656fc53d59f4b Mon Sep 17 00:00:00 2001 From: jievince Date: Wed, 12 Aug 2020 19:28:31 +0800 Subject: [PATCH 05/17] fix port error --- tests/test_StorageClient.py | 2 +- tests/test_client.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 62c61c58..a2ca563c 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -66,7 +66,7 @@ def test_scanVertex(): assert result is not None and result.next().vertex_data is not None def test_scanPartEdge(): - result = storageClient.scanPartEdge(spaceName, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) + result = storageClient.scanPartEdge(spaceName, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().edge_data is not None def test_scanPartVertex(): diff --git a/tests/test_client.py b/tests/test_client.py index 260a48cc..08aec83a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -25,7 +25,7 @@ def get_port(): - return 36159 + return 3699 def create_pool(port): From 359555a0bc3887ca5b450e9784343118e6ca302b Mon Sep 17 00:00:00 2001 From: jievince Date: Wed, 12 Aug 2020 19:35:56 +0800 Subject: [PATCH 06/17] fix none client assert --- tests/test_StorageClient.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index a2ca563c..75c615ff 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -33,8 +33,10 @@ def prepare(): connection_pool = ConnectionPool(host, graph_port) client = GraphClient(connection_pool) - if client.is_none(): - raise AuthException('Connect failed') + if client is None: + print('Error: None GraphClient') + assert False + return resp = client.authenticate('user', 'password') assert resp.error_code == 0, resp.error_msg resp = client.execute('DROP SPACE IF EXISTS %s' % spaceName) From 99326c3b62f23d588cbb8daf71e7d41515cec5d4 Mon Sep 17 00:00:00 2001 From: jievince Date: Wed, 12 Aug 2020 20:20:16 +0800 Subject: [PATCH 07/17] update test_StorageClient.py --- tests/test_StorageClient.py | 60 ++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 75c615ff..eff2d4b5 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -30,34 +30,39 @@ from nebula.ngStorage.StorageClient import StorageClient from nebula.ngMeta.MetaClient import MetaClient -def prepare(): - connection_pool = ConnectionPool(host, graph_port) - client = GraphClient(connection_pool) - if client is None: - print('Error: None GraphClient') + +def test_prepare(): + try: + client = GraphClient(ConnectionPool(host, graph_port)) + if client is None: + print('Error: None GraphClient') + assert False + return + resp = client.authenticate('user', 'password') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('DROP SPACE IF EXISTS %s' % spaceName) + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE SPACE %s(partition_num=1)' % spaceName) + assert resp.error_code == 0, resp.error_msg + time.sleep(5) + resp = client.execute('USE %s' % spaceName) + assert resp.error_code == 0, resp.error_msg + time.sleep(5) + resp = client.execute('CREATE TAG player(name string, age int)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE EDGE follow(degree double)') + assert resp.error_code == 0, resp.error_msg + time.sleep(12) + resp = client.execute('INSERT VERTEX player(name, age) VALUES 1:(\'Bob\', 18)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX player(name, age) VALUES 2:(\'Tome\', 22)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT EDGE follow(degree) VALUES 1->2:(94.7)') + assert resp.error_code == 0, resp.error_msg + except Exception as ex: + print(ex) + client.sign_out() assert False - return - resp = client.authenticate('user', 'password') - assert resp.error_code == 0, resp.error_msg - resp = client.execute('DROP SPACE IF EXISTS %s' % spaceName) - assert resp.error_code == 0, resp.error_msg - resp = client.execute('CREATE SPACE %s(partition_num=1)' % spaceName) - assert resp.error_code == 0, resp.error_msg - time.sleep(5) - resp = client.execute('USE %s' % spaceName) - assert resp.error_code == 0, resp.error_msg - time.sleep(5) - resp = client.execute('CREATE TAG player(name string, age int)') - assert resp.error_code == 0, resp.error_msg - resp = client.execute('CREATE EDGE follow(degree double)') - assert resp.error_code == 0, resp.error_msg - time.sleep(12) - resp = client.execute('INSERT VERTEX player(name, age) VALUES 1:(\'Bob\', 18)') - assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT VERTEX player(name, age) VALUES 2:(\'Tome\', 22)') - assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT EDGE follow(degree) VALUES 1->2:(94.7)') - assert resp.error_code == 0, resp.error_msg def test_scanEdge(): result = storageClient.scanEdge(spaceName, {'follow':['degree']}, True, 100, 0, sys.maxsize) @@ -113,7 +118,6 @@ def test_handleResultCodes(): graph_port = 3699 storage_port = 44500 spaceName = 'test_storage' -prepare() metaClient = MetaClient([('127.0.0.1', meta_port)]) metaClient.connect() storageClient = StorageClient(metaClient) From 8763f69d9d6fdff6c633a13ac1d18bf56bb4bc08 Mon Sep 17 00:00:00 2001 From: jievince Date: Wed, 12 Aug 2020 20:39:01 +0800 Subject: [PATCH 08/17] update test_StorageClient.py --- tests/test_StorageClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index eff2d4b5..817b004f 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -118,6 +118,6 @@ def test_handleResultCodes(): graph_port = 3699 storage_port = 44500 spaceName = 'test_storage' -metaClient = MetaClient([('127.0.0.1', meta_port)]) +metaClient = MetaClient([(host, meta_port)]) metaClient.connect() storageClient = StorageClient(metaClient) From a333e3620de1c7b224687d0c95c96248d372074f Mon Sep 17 00:00:00 2001 From: jievince Date: Sun, 23 Aug 2020 00:55:19 +0800 Subject: [PATCH 09/17] add timestamp --- examples/StorageClientExample.py | 46 ++++++++++--------- nebula/ngData/data.py | 30 +++++++----- nebula/ngMeta/MetaClient.py | 32 +++++-------- nebula/ngStorage/StorageClient.py | 33 +++++++------ .../ngProcessor/ScanEdgeProcessor.py | 4 +- .../ngProcessor/ScanVertexProcessor.py | 4 +- tests/test_StorageClient.py | 2 +- 7 files changed, 75 insertions(+), 76 deletions(-) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index 3f1a86ca..0c020b4f 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. @@ -10,6 +10,7 @@ """ import sys, getopt +from meta.ttypes import ErrorCode sys.path.insert(0, '../') @@ -23,9 +24,8 @@ def scanEdge(space, returnCols, allCols): scanEdgeResponseIter = storageClient.scanEdge(space, returnCols, allCols, 100, 0, sys.maxsize) scanEdgeResponse = scanEdgeResponseIter.next() if scanEdgeResponse is None: - print('not ok') - else: - print('ok') + print('scanEdgeResponse is None') + return processEdge(space, scanEdgeResponse) while scanEdgeResponseIter.hasNext(): scanEdgeResponse = scanEdgeResponseIter.next() @@ -38,9 +38,8 @@ def scanVertex(space, returnCols, allCols): scanVertexResponseIter = storageClient.scanVertex(space, returnCols, allCols, 100, 0, sys.maxsize) scanVertexResponse = scanVertexResponseIter.next() if scanVertexResponse is None: - print('not ok') - else: - print('ok') + print('scanVertexResponse is None') + return processVertex(space, scanVertexResponse) while scanVertexResponseIter.hasNext(): scanVertexResponse = scanVertexResponseIter.next() @@ -53,11 +52,10 @@ def processEdge(space, scanEdgeResponse): result = scanEdgeProcessor.process(space, scanEdgeResponse) # Get the corresponding rows by edgeName for edgeName, edgeRows in result.rows.items(): - print('edgeName: ', edgeName) for row in edgeRows: + srcId = row.defaultProperties[0].getValue() dstId = row.defaultProperties[2].getValue() - print('srcId: ', srcId, ' dstId: ', dstId) props = {} for prop in row.properties: propName = prop.getName() @@ -68,13 +66,10 @@ def processEdge(space, scanEdgeResponse): def processVertex(space, scanVertexResponse): result = scanVertexProcessor.process(space, scanVertexResponse) if result is None: - print('processVertex: result is None') return None for tagName, tagRows in result.rows.items(): - print('tagName: ', tagName) for row in tagRows: vid = row.defaultProperties[0].getValue() - print('vid: ', vid) props = {} for prop in row.properties: propName = prop.getName() @@ -85,21 +80,29 @@ def processVertex(space, scanVertexResponse): def getReturnCols(space): tagItems = metaClient.getTags(space) vertexReturnCols = {} - for tagItem in tagItems: - tagName = tagItem.tag_name - vertexReturnCols[tagName] = metaClient.getTagSchema(space, tagName).keys() + if tagItems is None: + print('tags not found in space ', space) + else: + for tagItem in tagItems: + tagName = tagItem.tag_name + vertexReturnCols[tagName] = metaClient.getTagSchema(space, tagName).keys() edgeItems = metaClient.getEdges(space) edgeReturnCols = {} - for edgeItem in edgeItems: - edgeName = edgeItem.edge_name - edgeReturnCols[edgeName] = metaClient.getEdgeSchema(space, edgeName).keys() + if edgeItems is None: + print('edges not found in space ', space) + else: + for edgeItem in edgeItems: + edgeName = edgeItem.edge_name + edgeReturnCols[edgeName] = metaClient.getEdgeSchema(space, edgeName).keys() return vertexReturnCols, edgeReturnCols if __name__ == '__main__': metaClient = MetaClient([(sys.argv[1], sys.argv[2])]) - metaClient.connect() + code = metaClient.connect() + if code == ErrorCode.E_FAIL_TO_CONNECT: + raise Exception('connect to %s:%d failed' % (sys.argv[1], sys.argv[2])) storageClient = StorageClient(metaClient) scanEdgeProcessor = ScanEdgeProcessor(metaClient) scanVertexProcessor = ScanVertexProcessor(metaClient) @@ -107,10 +110,9 @@ def getReturnCols(space): spaceToRead = sys.argv[3] vertexReturnCols, edgeReturnCols = getReturnCols(spaceToRead) allCols = True - + if spaceToRead not in metaClient.getPartsAllocFromCache().keys(): - raise Exception('spaceToRead %s is not found in nebula') + raise Exception('spaceToRead %s is not found in nebula' % spaceToRead) else: - print('scaning space %s' % spaceToRead) scanVertex(spaceToRead, vertexReturnCols, allCols) scanEdge(spaceToRead, edgeReturnCols, allCols) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 8f0cad78..647a89ea 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. @@ -11,7 +11,7 @@ class PropertyDef: PropertyType = Enum('PropertyType', ('UNKNOWN', 'BOOL', 'INT', 'VID', 'FLOAT', 'DOUBLE', \ - 'STRING', 'VERTEX_ID', 'TAG_ID', 'SRC_ID', 'EDGE_TYPE', 'EDGE_RANK', 'DST_ID')) + 'STRING', 'TIMESTAMP', 'VERTEX_ID', 'TAG_ID', 'SRC_ID', 'EDGE_TYPE', 'EDGE_RANK', 'DST_ID')) def __init__(self, propertyType, name): self.propertyType = propertyType @@ -22,15 +22,15 @@ class Property: def __init__(self, propertyType, name, value): self.propertyDef = PropertyDef(propertyType, name) self.value = value - + def getType(self): return self.propertyDef.type - + def getName(self): return self.propertyDef.name - + def getValue(self): - return self.value + return self.value class Row: def __init__(self, defaultProperties, properties): @@ -47,9 +47,12 @@ def __init__(self, schema, schemaVersion=0): self.propertyTypes = [] self.offset = 0 + propTypeMap = {0: PropertyDef.PropertyType.UNKNOWN, 1: PropertyDef.PropertyType.BOOL, 2: PropertyDef.PropertyType.INT, + 3: PropertyDef.PropertyType.VID, 4: PropertyDef.PropertyType.FLOAT, 5: PropertyDef.PropertyType.DOUBLE, + 6: PropertyDef.PropertyType.STRING, 21: PropertyDef.PropertyType.TIMESTAMP} idx = 0 for columnDef in schema.columns: - propertyType = PropertyDef.PropertyType(columnDef.type.type+1) # ColumnDef is in common/ttypes.py + propertyType = propTypeMap[columnDef.type.type] # ColumnDef is in common/ttypes.py columnName = columnDef.name self.defs.append((columnName, propertyType)) self.propertyNameIndex[columnName] = idx @@ -76,7 +79,6 @@ def decodeValue(self, value, schemaVersion=None): else: ver |= value[self.offset] << 8 self.offSet += 1 - # print('blkOffBytesNum: ', blkOffBytesNum, ' verBytesNum: ', verBytesNum, ' ver: ', ver, ' schemaVersion: ', schemaVersion) if ver != schemaVersion: raise Exception('parsed version %d is not equal to version %d provided', ver, schemaVersion) self.offset += blkOffBytesNum * (self.fieldNum // 16) @@ -94,6 +96,8 @@ def decodeValue(self, value, schemaVersion=None): properties.append(self.getDoubleProperty(field, value)) elif propertyType == PropertyDef.PropertyType.STRING: properties.append(self.getStringProperty(field, value)) + elif propertyType == PropertyDef.PropertyType.TIMESTAMP: + properties.append(self.getTimeStampProperty(field, value)) else: raise Exception('Invalid propertyType in schema: ', propertyType) @@ -134,6 +138,11 @@ def getIntProperty(self, name, value): val = self.readCompressedInt(value) return Property(PropertyDef.PropertyType.INT, name, val) #### 字节流解析出data + def getTimeStampProperty(self, name, value): + val = self.readCompressedInt(value) + # val = datetime.fromtimestamp(val) + return Property(PropertyDef.PropertyType.TIMESTAMP, name, val) + def getFloatProperty(self, name, value): val = struct.unpack_from('= 0: break val |= (byteV & 0x7f) << shift @@ -171,7 +178,6 @@ def readCompressedInt(self, value): return None val |= byteV << shift curOff += 1 - #print('readCompressedInt: ', value[self.offset:curOff], 'val is: ', val) self.offset = curOff return val diff --git a/nebula/ngMeta/MetaClient.py b/nebula/ngMeta/MetaClient.py index 185d18b8..ef191cb8 100644 --- a/nebula/ngMeta/MetaClient.py +++ b/nebula/ngMeta/MetaClient.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. @@ -42,10 +42,9 @@ class RepeatTimer(Timer): def __init__(self, interval, function): Timer.__init__(self, interval, function) self.daemon = True # set the RepeatTimer thread as a daemon thread, so it can end when main thread ends - + def run(self): while not self.finished.wait(self.interval): - #print('daemon: ', self.daemon) self.function(*self.args, **self.kwargs) class MetaClient: @@ -63,7 +62,7 @@ def __init__(self, addresses, timeout=1000, self.tagNameMap = {} # map> self.edgeNameMap = {} # map> self.client = None - + def connect(self): while self.connectionRetry > 0: code = self.doConnect(self.addresses) @@ -74,7 +73,6 @@ def connect(self): def doConnect(self, addresses): address = addresses[random.randint(0, len(addresses)-1)] - #print('metaClient is connecting to: ', address[0], address[1]) host = address[0] port = address[1] tTransport = TSocket.TSocket(host, port) @@ -85,15 +83,13 @@ def doConnect(self, addresses): self.client = Client(tProtocol) self.updateSchemas() RepeatTimer(2, self.updateSchemas).start() # call updatSchemas() every 2 seconds - + return 0 def updateSchemas(self): - print('threading active_count: ', threading.active_count()) for spaceIdName in self.listSpaces(): spaceName = spaceIdName.name # class IdName self.spaceNameMap[spaceName] = spaceIdName.id.get_space_id() - #print(spaceName, spaceIdName.id.get_space_id()) self.spacePartLocation[spaceName] = self.getPartsAlloc(spaceName) self.spacePartLeader[spaceName] = {} # Loading tag schema's cache @@ -129,7 +125,7 @@ def getSpacePartLeaderFromCache(self, spaceName, partId): if partId not in self.spacePartLeader[spaceName].keys(): return None return self.spacePartLeader[spaceName][partId] - + def updateSpacePartLeader(self, spaceName, partId, leader): self.spacePartLeader[spaceName][partId] = leader @@ -141,7 +137,7 @@ def setSpacePartLeader(self): return None for hostItem in listHostsResp.hosts: - host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostItem.hostAddr.ip & 0xffffffff))) + host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostItem.hostAddr.ip & 0xffffffff))) port = hostItem.hostAddr.port leader = (host, port) for space, partIds in hostItem.leader_parts.items(): @@ -152,7 +148,7 @@ def listSpaces(self): listSpacesReq = ListSpacesReq() listSpacesResp = self.client.listSpaces(listSpacesReq) if listSpacesResp.code == ErrorCode.SUCCEEDED: - return listSpacesResp.spaces########## spaceNameID--> IdName + return listSpacesResp.spaces # spaceNameID--> IdName else: print('list spaces error, error code: ', listSpacesResp.code) return None @@ -195,13 +191,13 @@ def getPartAllocFromCache(self, spaceName, part): partsAlloc = self.spacePartLocation[spaceName] if part in partsAlloc.keys(): return partsAlloc[part] - + return None - + def getTagItemFromCache(self, spaceName, tagName): if spaceName in self.spaceTagItems.keys() and tagName in self.spaceTagItems[spaceName].keys(): return self.spaceTagItems[spaceName][tagName] - + return None def getTagNameFromCache(self, spaceName, tagId): @@ -218,7 +214,7 @@ def getTags(self, spaceName): return None listTagsReq = ListTagsReq(spaceId) listTagsResp = self.client.listTags(listTagsReq) - + if listTagsResp.code == ErrorCode.SUCCEEDED: return listTagsResp.tags else: @@ -237,7 +233,6 @@ def getTag(self, spaceName, tagName, version=-1): def getTagSchema(self, spaceName, tagName, version=-1): spaceId = self.getSpaceIdFromCache(spaceName) - print('spaceId: ', spaceId) if spaceId == -1: return None getTagReq = GetTagReq(spaceId, tagName, version) @@ -245,7 +240,6 @@ def getTagSchema(self, spaceName, tagName, version=-1): result = {} for columnDef in getTagResp.schema.columns: result[columnDef.name] = columnDef.type.type - print('result: ', result) return result def getEdgeItemFromCache(self, spaceName, edgeName): @@ -260,7 +254,7 @@ def getEdgeItemFromCache(self, spaceName, edgeName): return edgeItems[edgeName] else: return None - + def getEdgeNameFromCache(self, spaceName, edgeType): if spaceName in self.edgeNameMap.keys(): edgeNames = self.edgeNameMap[spaceName] @@ -302,6 +296,4 @@ def getEdgeSchema(self, spaceName, edgeName, version=-1): result = {} for columnDef in getEdgeResp.schema.columns: result[columnDef.name] = columnDef.type.type - print('edge sapcId: ', spaceId) - print(result) return result diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index 76b065cc..3d25a8f5 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -26,7 +26,7 @@ def __init__(self, it=None): self.it = iter(it) self._hasnext = None - def __iter__(self): + def __iter__(self): return self def next(self): @@ -39,11 +39,11 @@ def next(self): def hasNext(self): if self._hasnext is None: - try: + try: self._thenext = next(self.it) - except StopIteration: + except StopIteration: self._hasnext = False - else: + else: self._hasnext = True return self._hasnext @@ -64,18 +64,18 @@ def hasNext(self): def next(self): self.scanEdgeRequest.cursor = self.cursor scanEdgeResponse = self.client.scanEdge(self.scanEdgeRequest) - assert(scanEdgeResponse is not None), 'scanEdgeReponse is none' + if scanEdgeResponse is None: + raise Exception('scanEdgeResponse is None') self.cursor = scanEdgeResponse.next_cursor self.haveNext = scanEdgeResponse.has_next if not self.clientDad.isSuccessfully(scanEdgeResponse): - print('scanEdgeResponse is not successfully, failed_codes: ', scanEdgeResponse.result.failed_codes) self.leader, self.client = self.clientDad.handleResultCodes(scanEdgeResponse.result.failed_codes, self.space) self.haveNext = False return None else: return scanEdgeResponse - + return None @@ -95,7 +95,8 @@ def hasNext(self): def next(self): self.scanVertexRequest.cursor = self.cursor scanVertexResponse = self.client.scanVertex(self.scanVertexRequest) - assert(scanVertexResponse is not None), 'scanVertexReponse is none' + if scanVertexResponse is None: + raise Exception('scanVertexReponse is None') self.cursor = scanVertexResponse.next_cursor self.haveNext = scanVertexResponse.has_next @@ -140,7 +141,7 @@ def next(self): scanEdgeRequest = ScanEdgeRequest(spaceId, part, None, colums, self.allCols, self.limit, self.startTime, self.endTime) self.scanEdgeResponseIter = self.clientDad.doScanEdge(self.space, leader, scanEdgeRequest) assert(self.scanEdgeResponseIter is not None), 'scanEdgeResponseIter is None' - + return self.scanEdgeResponseIter.next() @@ -164,7 +165,6 @@ def next(self): part = self.partIdsIter.next() if part is None: return None - print('part: ', part) leader = self.clientDad.getLeader(self.space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, self.space)) @@ -205,7 +205,6 @@ def doConnects(self, addresses): def doConnect(self, address): host = address[0] port = address[1] - #print('StorageClient is connect to: tTransport ip: %s, port: %s' % (host, port)) tTransport = TSocket.TSocket(host, port) tTransport.setTimeout(self.timeout) tTransport = TTransport.TBufferedTransport(tTransport) @@ -217,7 +216,7 @@ def scanEdge(self, space, returnCols, allCols, limit, startTime, endTime): partIds = self.metaClient.getPartsAllocFromCache()[space].keys() partIdsIter = Iterator(partIds) return ScanSpaceEdgeResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) - + def scanPartEdge(self, space, part, returnCols, allCols, limit, startTime, endTime): spaceId = self.metaClient.getSpaceIdFromCache(space) columns = self.getEdgeReturnCols(space, returnCols) @@ -231,7 +230,7 @@ def scanVertex(self, space, returnCols, allCols, limit, startTime, endTime): partIds = self.metaClient.getPartsAllocFromCache()[space].keys() partIdsIter = Iterator(partIds) return ScanSpaceVertexResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) - + def scanPartVertex(self, space, part, returnCols, allCols, limit, startTime, endTime): spaceId = self.metaClient.getSpaceIdFromCache(space) columns = self.getVertexReturnCols(space, returnCols) @@ -249,7 +248,7 @@ def doScanEdge(self, space, leader, scanEdgeRequest): return None return ScanEdgeResponseIter(self, space, leader, scanEdgeRequest, client) - + def doScanVertex(self, space, leader, scanVertexRequest): client = self.connect(leader) if client is None: @@ -273,7 +272,7 @@ def getEdgeReturnCols(self, space, returnCols): propDefs.append(propDef) columns[edgeType] = propDefs return columns - + def getVertexReturnCols(self, space, returnCols): columns = {} for tagName, propNames in returnCols.items(): @@ -291,7 +290,7 @@ def getVertexReturnCols(self, space, returnCols): def getLeader(self, spaceName, part): return self.metaClient.getSpacePartLeaderFromCache(spaceName, part) - + def updateLeader(self, spaceName, partId, leader): self.metaClient.updateSpacePartLeader(spaceName, partId, leader) @@ -310,7 +309,7 @@ def handleResultCodes(self, failedCodes, space): else: newClient = None return newLeader, newClient - + return None, None def isSuccessfully(self, response): diff --git a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py index 3c6924ef..5843ee59 100644 --- a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. @@ -27,7 +27,7 @@ def process(self, spaceName, scanEdgeResponse): edgeItem = self.metaClient.getEdgeItemFromCache(spaceName, edgeName) schemaVersion = edgeItem.version rowReaders[edgeType] = RowReader(schema, schemaVersion) - rows[edgeName] = [] ### + rows[edgeName] = [] edgeTypeNameMap[edgeType] = edgeName else: print('scanEdgeResponse.edge_schema is None') diff --git a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py index e9413b95..90612dfa 100644 --- a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. @@ -29,7 +29,7 @@ def process(self, spaceName, scanVertexResponse): tagItem = self.metaClient.getTagItemFromCache(spaceName, tagName) schemaVersion = tagItem.version rowReaders[tagId] = RowReader(schema, schemaVersion) - rows[tagName] = [] ### + rows[tagName] = [] tagIdNameMap[tagId] = tagName else: print('scanVertexResponse.vertex_schema is None') diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 817b004f..9a79b2c0 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -1,6 +1,6 @@ # --coding:utf-8-- # -# Copyright (c) 2019 vesoft inc. All rights reserved. +# Copyright (c) 2020 vesoft inc. All rights reserved. # # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. From e84d89ce66a57e94680a186fea0fe74773a1cf89 Mon Sep 17 00:00:00 2001 From: jievince Date: Tue, 25 Aug 2020 10:46:02 +0800 Subject: [PATCH 10/17] Make the code conform to the variable naming convention --- examples/StorageClientExample.py | 132 +++--- nebula/ngData/data.py | 190 ++++----- nebula/ngMeta/MetaClient.py | 326 +++++++-------- nebula/ngStorage/StorageClient.py | 392 +++++++++--------- .../ngProcessor/ScanEdgeProcessor.py | 50 +-- .../ngProcessor/ScanVertexProcessor.py | 52 +-- tests/test_StorageClient.py | 70 ++-- 7 files changed, 607 insertions(+), 605 deletions(-) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index 0c020b4f..690670b2 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -20,99 +20,99 @@ from nebula.ngStorage.ngProcessor.ScanVertexProcessor import ScanVertexProcessor -def scanEdge(space, returnCols, allCols): - scanEdgeResponseIter = storageClient.scanEdge(space, returnCols, allCols, 100, 0, sys.maxsize) - scanEdgeResponse = scanEdgeResponseIter.next() - if scanEdgeResponse is None: - print('scanEdgeResponse is None') +def scan_edge(space, return_cols, all_cols): + scan_edge_response_iter = storage_client.scan_edge(space, return_cols, all_cols, 100, 0, sys.maxsize) + scan_edge_response = scan_edge_response_iter.next() + if scan_edge_response is None: + print('scan_edge_response is None') return - processEdge(space, scanEdgeResponse) - while scanEdgeResponseIter.hasNext(): - scanEdgeResponse = scanEdgeResponseIter.next() - if scanEdgeResponse is None: + process_edge(space, scan_edge_response) + while scan_edge_response_iter.has_next(): + scan_edge_response = scan_edge_response_iter.next() + if scan_edge_response is None: print("Error occurs while scaning edge") break - processEdge(space, scanEdgeResponse) + process_edge(space, scan_edge_response) -def scanVertex(space, returnCols, allCols): - scanVertexResponseIter = storageClient.scanVertex(space, returnCols, allCols, 100, 0, sys.maxsize) - scanVertexResponse = scanVertexResponseIter.next() - if scanVertexResponse is None: - print('scanVertexResponse is None') +def scan_vertex(space, return_cols, all_cols): + scan_vertex_response_iter = storage_client.scan_vertex(space, return_cols, all_cols, 100, 0, sys.maxsize) + scan_vertex_response = scan_vertex_response_iter.next() + if scan_vertex_response is None: + print('scan_vertex_vesponse is None') return - processVertex(space, scanVertexResponse) - while scanVertexResponseIter.hasNext(): - scanVertexResponse = scanVertexResponseIter.next() - if scanVertexResponse is None: + process_vertex(space, scan_vertex_response) + while scan_vertex_response_iter.has_next(): + scan_vertex_response = scan_vertex_response_iter.next() + if scan_vertex_response is None: print("Error occurs while scaning vertex") break - processVertex(space, scanVertexResponse) + process_vertex(space, scan_vertex_response) -def processEdge(space, scanEdgeResponse): - result = scanEdgeProcessor.process(space, scanEdgeResponse) +def process_edge(space, scan_edge_response): + result = scan_edge_processor.process(space, scan_edge_response) # Get the corresponding rows by edgeName - for edgeName, edgeRows in result.rows.items(): - for row in edgeRows: - - srcId = row.defaultProperties[0].getValue() - dstId = row.defaultProperties[2].getValue() + for edge_name, edge_rows in result._rows.items(): + for row in edge_rows: + srcId = row._default_properties[0].get_value() + dstId = row._default_properties[2].get_value() + print(srcId,'->', dstId) props = {} - for prop in row.properties: - propName = prop.getName() - propValue = prop.getValue() - props[propName] = propValue + for prop in row._properties: + prop_name = prop.get_name() + prop_value = prop.get_value() + props[prop_name] = prop_value print(props) -def processVertex(space, scanVertexResponse): - result = scanVertexProcessor.process(space, scanVertexResponse) +def process_vertex(space, scan_vertex_response): + result = scan_vertex_processor.process(space, scan_vertex_response) if result is None: return None - for tagName, tagRows in result.rows.items(): - for row in tagRows: - vid = row.defaultProperties[0].getValue() + for tag_name, tag_rows in result._rows.items(): + for row in tag_rows: + vid = row._default_properties[0].get_value() props = {} - for prop in row.properties: - propName = prop.getName() - propValue = prop.getValue() - props[propName] = propValue + for prop in row._properties: + prop_name = prop.get_name() + prop_value = prop.get_value() + props[prop_name] = prop_value print(props) -def getReturnCols(space): - tagItems = metaClient.getTags(space) - vertexReturnCols = {} - if tagItems is None: +def get_return_cols(space): + tag_items = meta_client.get_tags(space) + vertex_return_cols = {} + if tag_items is None: print('tags not found in space ', space) else: - for tagItem in tagItems: - tagName = tagItem.tag_name - vertexReturnCols[tagName] = metaClient.getTagSchema(space, tagName).keys() - edgeItems = metaClient.getEdges(space) - edgeReturnCols = {} - if edgeItems is None: + for tag_item in tag_items: + tag_name = tag_item.tag_name + vertex_return_cols[tag_name] = meta_client.get_tag_schema(space, tag_name).keys() + edge_items = meta_client.get_edges(space) + edge_return_cols = {} + if edge_items is None: print('edges not found in space ', space) else: - for edgeItem in edgeItems: - edgeName = edgeItem.edge_name - edgeReturnCols[edgeName] = metaClient.getEdgeSchema(space, edgeName).keys() + for edge_item in edge_items: + edge_name = edge_item.edge_name + edge_return_cols[edge_name] = meta_client.get_edge_schema(space, edge_name).keys() - return vertexReturnCols, edgeReturnCols + return vertex_return_cols, edge_return_cols if __name__ == '__main__': - metaClient = MetaClient([(sys.argv[1], sys.argv[2])]) - code = metaClient.connect() + meta_client = MetaClient([(sys.argv[1], sys.argv[2])]) + code = meta_client.connect() if code == ErrorCode.E_FAIL_TO_CONNECT: raise Exception('connect to %s:%d failed' % (sys.argv[1], sys.argv[2])) - storageClient = StorageClient(metaClient) - scanEdgeProcessor = ScanEdgeProcessor(metaClient) - scanVertexProcessor = ScanVertexProcessor(metaClient) + storage_client = StorageClient(meta_client) + scan_edge_processor = ScanEdgeProcessor(meta_client) + scan_vertex_processor = ScanVertexProcessor(meta_client) - spaceToRead = sys.argv[3] - vertexReturnCols, edgeReturnCols = getReturnCols(spaceToRead) - allCols = True + space_to_read = sys.argv[3] + vertex_return_cols, edge_return_cols = get_return_cols(space_to_read) + all_cols = True - if spaceToRead not in metaClient.getPartsAllocFromCache().keys(): - raise Exception('spaceToRead %s is not found in nebula' % spaceToRead) + if space_to_read not in meta_client.get_parts_alloc_from_cache().keys(): + raise Exception('spaceToRead %s is not found in nebula' % space_to_read) else: - scanVertex(spaceToRead, vertexReturnCols, allCols) - scanEdge(spaceToRead, edgeReturnCols, allCols) + scan_vertex(space_to_read, vertex_return_cols, all_cols) + scan_edge(space_to_read, edge_return_cols, all_cols) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 647a89ea..46367d1d 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -13,159 +13,159 @@ class PropertyDef: PropertyType = Enum('PropertyType', ('UNKNOWN', 'BOOL', 'INT', 'VID', 'FLOAT', 'DOUBLE', \ 'STRING', 'TIMESTAMP', 'VERTEX_ID', 'TAG_ID', 'SRC_ID', 'EDGE_TYPE', 'EDGE_RANK', 'DST_ID')) - def __init__(self, propertyType, name): - self.propertyType = propertyType - self.name = name + def __init__(self, property_type, name): + self._property_type = property_type + self._name = name class Property: - def __init__(self, propertyType, name, value): - self.propertyDef = PropertyDef(propertyType, name) - self.value = value + def __init__(self, property_type, name, value): + self._propertyDef = PropertyDef(property_type, name) + self._value = value - def getType(self): - return self.propertyDef.type + def get_type(self): + return self._propertyDef._property_type - def getName(self): - return self.propertyDef.name + def get_name(self): + return self._propertyDef._name - def getValue(self): - return self.value + def get_value(self): + return self._value class Row: - def __init__(self, defaultProperties, properties): - self.defaultProperties = defaultProperties - self.properties = properties + def __init__(self, default_properties, properties): + self._default_properties = default_properties + self._properties = properties class RowReader: - def __init__(self, schema, schemaVersion=0): - self.schemaVersion = schemaVersion - self.defs = [] - self.fieldNum = 0 - self.propertyNameIndex = {} - self.propertyTypes = [] - self.offset = 0 - - propTypeMap = {0: PropertyDef.PropertyType.UNKNOWN, 1: PropertyDef.PropertyType.BOOL, 2: PropertyDef.PropertyType.INT, + def __init__(self, schema, schema_version=0): + self._schema_version = schema_version + self._defs = [] + self._field_num = 0 + self._property_name_index = {} + self._property_types = [] + self._offset = 0 + + prop_type_map = {0: PropertyDef.PropertyType.UNKNOWN, 1: PropertyDef.PropertyType.BOOL, 2: PropertyDef.PropertyType.INT, 3: PropertyDef.PropertyType.VID, 4: PropertyDef.PropertyType.FLOAT, 5: PropertyDef.PropertyType.DOUBLE, 6: PropertyDef.PropertyType.STRING, 21: PropertyDef.PropertyType.TIMESTAMP} idx = 0 - for columnDef in schema.columns: - propertyType = propTypeMap[columnDef.type.type] # ColumnDef is in common/ttypes.py - columnName = columnDef.name - self.defs.append((columnName, propertyType)) - self.propertyNameIndex[columnName] = idx + for column_def in schema.columns: + property_type = prop_type_map[column_def.type.type] # ColumnDef is in common/ttypes.py + column_name = column_def.name + self._defs.append((column_name, property_type)) + self._property_name_index[column_name] = idx idx += 1 - self.fieldNum = len(self.defs) + self._field_num = len(self._defs) - def decodeValue(self, value, schemaVersion=None): - if schemaVersion is None: - schemaVersion = self.schemaVersion - self.offset = 0 + def decode_value(self, value, schema_version=None): + if schema_version is None: + schema_version = self._schema_version + self._offset = 0 # need to check if value is valid if six.PY2: - blkOffBytesNum = ord(value[0]) & 0x07 + 1 - verBytesNum = ord(value[0]) >> 5 + blk_off_bytes_num = ord(value[0]) & 0x07 + 1 + ver_bytes_num = ord(value[0]) >> 5 else: - blkOffBytesNum = value[0] & 0x07 + 1 - verBytesNum = value[0] >> 5 - self.offset += 1 + blk_off_bytes_num = value[0] & 0x07 + 1 + ver_bytes_num = value[0] >> 5 + self._offset += 1 ver = 0 - if verBytesNum > 0: - for i in range(verBytesNum): + if ver_bytes_num > 0: + for i in range(ver_bytes_num): if six.PY2: - ver |= ord(value[self.offset]) << 8 + ver |= ord(value[self._offset]) << 8 else: - ver |= value[self.offset] << 8 - self.offSet += 1 - if ver != schemaVersion: - raise Exception('parsed version %d is not equal to version %d provided', ver, schemaVersion) - self.offset += blkOffBytesNum * (self.fieldNum // 16) + ver |= value[self._offset] << 8 + self._offSet += 1 + if ver != schema_version: + raise Exception('parsed version %d is not equal to version %d provided', ver, schema_version) + self._offset += blk_off_bytes_num * (self._field_num // 16) properties = [] - for i in range(len(self.defs)): - field = self.defs[i][0] - propertyType = self.defs[i][1] - if propertyType == PropertyDef.PropertyType.BOOL: - properties.append(self.getBoolProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.INT: - properties.append(self.getIntProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.FLOAT: #unused now - properties.append(self.getFloatProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.DOUBLE: - properties.append(self.getDoubleProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.STRING: - properties.append(self.getStringProperty(field, value)) - elif propertyType == PropertyDef.PropertyType.TIMESTAMP: - properties.append(self.getTimeStampProperty(field, value)) + for i in range(len(self._defs)): + field = self._defs[i][0] + property_type = self._defs[i][1] + if property_type == PropertyDef.PropertyType.BOOL: + properties.append(self.get_bool_property(field, value)) + elif property_type == PropertyDef.PropertyType.INT: + properties.append(self.get_int_property(field, value)) + elif property_type == PropertyDef.PropertyType.FLOAT: #unused now + properties.append(self.get_float_property(field, value)) + elif property_type == PropertyDef.PropertyType.DOUBLE: + properties.append(self.get_double_property(field, value)) + elif property_type == PropertyDef.PropertyType.STRING: + properties.append(self.get_string_property(field, value)) + elif property_type == PropertyDef.PropertyType.TIMESTAMP: + properties.append(self.get_timestamp_property(field, value)) else: - raise Exception('Invalid propertyType in schema: ', propertyType) + raise Exception('Invalid propertyType in schema: ', property_type) return properties - def edgeKey(self, srcId, edgeType, dstId): + def edge_key(self, srcId, edgeType, dstId): properties = [] properties.append(Property(PropertyDef.PropertyType.SRC_ID, "_srcId", srcId)) properties.append(Property(PropertyDef.PropertyType.EDGE_TYPE, "_edgeType", edgeType)) properties.append(Property(PropertyDef.PropertyType.DST_ID, "_dstId", dstId)) return properties - def vertexKey(self, vertexId, tagId): + def vertex_key(self, vertexId, tagId): properties = [] properties.append(Property(PropertyDef.PropertyType.VERTEX_ID, "_vertexId", vertexId)) properties.append(Property(PropertyDef.PropertyType.TAG_ID, "_tagId", tagId)) return properties - def getProperty(self, row, name): - if name not in propertyNameIndex.keys(): + def get_property(self, row, name): + if name not in property_name_index.keys(): return None - return row.properties[propertyNameIndex[name]] + return row.properties[property_name_index[name]] - def getPropertyByIndex(self, row, index): - if index < 0 or index >= len(row.getProperties()): + def get_property_by_index(self, row, index): + if index < 0 or index >= len(row.get_properties()): return None return row.properties[index] - def getBoolProperty(self, name, value): + def get_bool_property(self, name, value): if six.PY2: - val = ord(value[self.offset]) != 0x00 + val = ord(value[self._offset]) != 0x00 else: - val = value[self.offset] != 0x00 - self.offset += 1 + val = value[self._offset] != 0x00 + self._offset += 1 return Property(PropertyDef.PropertyType.BOOL, name, val) - def getIntProperty(self, name, value): - val = self.readCompressedInt(value) + def get_int_property(self, name, value): + val = self.read_compressed_int(value) return Property(PropertyDef.PropertyType.INT, name, val) #### 字节流解析出data - def getTimeStampProperty(self, name, value): - val = self.readCompressedInt(value) + def get_timestamp_property(self, name, value): + val = self.read_compressed_int(value) # val = datetime.fromtimestamp(val) return Property(PropertyDef.PropertyType.TIMESTAMP, name, val) - def getFloatProperty(self, name, value): - val = struct.unpack_from(' - self.spacePartLocation = {} # map>> - self.spacePartLeader = {} # map> - self.spaceTagItems = {} # map> - self.spaceEdgeItems = {} # map> - self.tagNameMap = {} # map> - self.edgeNameMap = {} # map> - self.client = None + connection_retry=3, execution_retry=3): + self._addresses = addresses + self._timeout = timeout + self._connection_retry = connection_retry + self._execution_retry = execution_retry + self._space_name_map = {} # map + self._space_part_location = {} # map>> + self._space_part_leader = {} # map> + self._space_tag_items = {} # map> + self._space_edge_items = {} # map> + self._tag_name_map = {} # map> + self._edge_name_map = {} # map> + self._client = None def connect(self): - while self.connectionRetry > 0: - code = self.doConnect(self.addresses) + while self._connection_retry > 0: + code = self.do_connect(self._addresses) if code == 0: return ErrorCode.SUCCEEDED - self.connectionRetry -= 1 + self._connection_retry -= 1 return ErrorCode.E_FAIL_TO_CONNECT - def doConnect(self, addresses): + def do_connect(self, addresses): address = addresses[random.randint(0, len(addresses)-1)] host = address[0] port = address[1] - tTransport = TSocket.TSocket(host, port) - tTransport.setTimeout(self.timeout) - tTransport = TTransport.TBufferedTransport(tTransport) - tProtocol = TBinaryProtocol.TBinaryProtocol(tTransport) - tTransport.open() - self.client = Client(tProtocol) - self.updateSchemas() - RepeatTimer(2, self.updateSchemas).start() # call updatSchemas() every 2 seconds + transport = TSocket.TSocket(host, port) + transport.setTimeout(self._timeout) + transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + transport.open() + self._client = Client(protocol) + self.update_schemas() + RepeatTimer(2, self.update_schemas).start() # call updatSchemas() every 2 seconds return 0 - def updateSchemas(self): - for spaceIdName in self.listSpaces(): - spaceName = spaceIdName.name # class IdName - self.spaceNameMap[spaceName] = spaceIdName.id.get_space_id() - self.spacePartLocation[spaceName] = self.getPartsAlloc(spaceName) - self.spacePartLeader[spaceName] = {} + def update_schemas(self): + for space_id_name in self.list_spaces(): + space_name = space_id_name.name # class IdName + self._space_name_map[space_name] = space_id_name.id.get_space_id() + self._space_part_location[space_name] = self.get_parts_alloc(space_name) + self._space_part_leader[space_name] = {} # Loading tag schema's cache tags = {} - tagsName = {} - for tagItem in self.getTags(spaceName): - tags[tagItem.tag_name] = tagItem - tagsName[tagItem.tag_id] = tagItem.tag_name + tags_name = {} + for tag_item in self.get_tags(space_name): + tags[tag_item.tag_name] = tag_item + tags_name[tag_item.tag_id] = tag_item.tag_name - self.spaceTagItems[spaceName] = tags - self.tagNameMap[spaceName] = tagsName + self._space_tag_items[space_name] = tags + self._tag_name_map[space_name] = tags_name # Loading edge schema's cache edges = {} - edgesName = {} - for edgeItem in self.getEdges(spaceName): - edges[edgeItem.edge_name] = edgeItem - edgesName[edgeItem.edge_type] = edgeItem.edge_name - self.spaceEdgeItems[spaceName] = edges - self.edgeNameMap[spaceName] = edgesName - self.setSpacePartLeader() + edges_name = {} + for edge_item in self.get_edges(space_name): + edges[edge_item.edge_name] = edge_item + edges_name[edge_item.edge_type] = edge_item.edge_name + self._space_edge_items[space_name] = edges + self._edge_name_map[space_name] = edges_name + self.set_space_part_leader() return 0 - def getSpaceIdFromCache(self, spaceName): - if spaceName not in self.spaceNameMap.keys(): + def get_space_id_from_cache(self, space_name): + if space_name not in self._space_name_map.keys(): return -1 else: - return self.spaceNameMap[spaceName] + return self._space_name_map[space_name] - def getSpacePartLeaderFromCache(self, spaceName, partId): - if spaceName not in self.spacePartLeader.keys(): + def get_space_part_leader_from_cache(self, space_name, part_id): + if space_name not in self._space_part_leader.keys(): return None - if partId not in self.spacePartLeader[spaceName].keys(): + if part_id not in self._space_part_leader[space_name].keys(): return None - return self.spacePartLeader[spaceName][partId] + return self._space_part_leader[space_name][part_id] - def updateSpacePartLeader(self, spaceName, partId, leader): - self.spacePartLeader[spaceName][partId] = leader + def update_space_part_leader(self, space_name, part_id, leader): + self._space_part_leader[space_name][part_id] = leader - def setSpacePartLeader(self): - listHostsReq = ListHostsReq() - listHostsResp = self.client.listHosts(listHostsReq) - if listHostsResp.code != ErrorCode.SUCCEEDED: - print('setSpacePartLeader error, eror code: ', listHostsResp.code) + def set_space_part_leader(self): + list_hosts_req = ListHostsReq() + list_hosts_resp = self._client.listHosts(list_hosts_req) + if list_hosts_resp.code != ErrorCode.SUCCEEDED: + print('set_space_part_leader error, eror code: ', list_hosts_resp.code) return None - for hostItem in listHostsResp.hosts: - host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostItem.hostAddr.ip & 0xffffffff))) - port = hostItem.hostAddr.port + for host_item in list_hosts_resp.hosts: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(host_item.hostAddr.ip & 0xffffffff))) + port = host_item.hostAddr.port leader = (host, port) - for space, partIds in hostItem.leader_parts.items(): - for partId in partIds: - self.spacePartLeader[space][partId] = leader - - def listSpaces(self): - listSpacesReq = ListSpacesReq() - listSpacesResp = self.client.listSpaces(listSpacesReq) - if listSpacesResp.code == ErrorCode.SUCCEEDED: - return listSpacesResp.spaces # spaceNameID--> IdName + for space, part_ids in host_item.leader_parts.items(): + for part_id in part_ids: + self._space_part_leader[space][part_id] = leader + + def list_spaces(self): + list_spaces_req = ListSpacesReq() + list_spaces_resp = self._client.listSpaces(list_spaces_req) + if list_spaces_resp.code == ErrorCode.SUCCEEDED: + return list_spaces_resp.spaces # space_nameID--> IdName else: - print('list spaces error, error code: ', listSpacesResp.code) + print('list spaces error, error code: ', list_spaces_resp.code) return None - def getPartFromCache(self, spaceName, part): - if spaceName not in self.spacePartLocation.keys(): - self.spacePartLocation[spaceName] = self.getPartsAlloc(spaceName) - partsAlloc = self.spacePartLocation[spaceName] - if partsAlloc is None or part not in partsAlloc.keys(): + def get_part_from_cache(self, space_name, part): + if space_name not in self._space_part_location.keys(): + self._space_part_location[space_name] = self.get_parts_alloc(space_name) + parts_alloc = self._space_part_location[space_name] + if parts_alloc is None or part not in parts_alloc.keys(): return None - return partsAlloc[part] + return parts_alloc[part] - def getPartsAlloc(self, spaceName): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_parts_alloc(self, space_name): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - getPartsAllocReq = GetPartsAllocReq(spaceId) - getPartsAllocResp = self.client.getPartsAlloc(getPartsAllocReq) + get_parts_alloc_req = GetPartsAllocReq(space_id) + get_parts_alloc_resp = self._client.getPartsAlloc(get_parts_alloc_req) - if getPartsAllocResp.code == ErrorCode.SUCCEEDED: - addressMap = {} - for partId, hostAddrs in getPartsAllocResp.parts.items(): + if get_parts_alloc_resp.code == ErrorCode.SUCCEEDED: + address_map = {} + for part_id, host_addrs in get_parts_alloc_resp.parts.items(): addresses = [] - for hostAddr in hostAddrs: - host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostAddr.ip & 0xffffffff))) - port = hostAddr.port + for host_addr in host_addrs: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(host_addr.ip & 0xffffffff))) + port = host_addr.port addresses.append((host, port)) - addressMap[partId] = addresses + address_map[part_id] = addresses - return addressMap + return address_map else: - print("get parts alloc error, error code: ", getPartsAllocResp.code) + print("get parts alloc error, error code: ", getParts_alloc_resp.code) return None - def getPartsAllocFromCache(self): - return self.spacePartLocation + def get_parts_alloc_from_cache(self): + return self._space_part_location - def getPartAllocFromCache(self, spaceName, part): - if spaceName in self.spacePartLocation.keys(): - partsAlloc = self.spacePartLocation[spaceName] - if part in partsAlloc.keys(): - return partsAlloc[part] + def get_part_alloc_from_cache(self, space_name, part): + if space_name in self._space_part_location.keys(): + parts_alloc = self._space_part_location[space_name] + if part in parts_alloc.keys(): + return parts_alloc[part] return None - def getTagItemFromCache(self, spaceName, tagName): - if spaceName in self.spaceTagItems.keys() and tagName in self.spaceTagItems[spaceName].keys(): - return self.spaceTagItems[spaceName][tagName] + def get_tag_item_from_cache(self, space_name, tag_name): + if space_name in self._space_tag_items.keys() and tag_name in self._space_tag_items[space_name].keys(): + return self._space_tag_items[space_name][tag_name] return None - def getTagNameFromCache(self, spaceName, tagId): - if spaceName in self.tagNameMap.keys(): - tagNames = self.tagNameMap[spaceName] - if tagId in tagNames.keys(): - return tagNames[tagId] + def get_tag_name_from_cache(self, space_name, tag_id): + if space_name in self._tag_name_map.keys(): + tag_names = self._tag_name_map[space_name] + if tag_id in tag_names.keys(): + return tag_names[tag_id] return None - def getTags(self, spaceName): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_tags(self, space_name): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - listTagsReq = ListTagsReq(spaceId) - listTagsResp = self.client.listTags(listTagsReq) + list_tags_req = ListTagsReq(space_id) + list_tags_resp = self._client.listTags(list_tags_req) - if listTagsResp.code == ErrorCode.SUCCEEDED: - return listTagsResp.tags + if list_tags_resp.code == ErrorCode.SUCCEEDED: + return list_tags_resp.tags else: - print('get tags error, error code: ', listTagsResp.code) + print('get tags error, error code: ', list_tags_resp.code) return None - def getTag(self, spaceName, tagName, version=-1): - spaceId = self.getSpaceIdFromCache(spaceName) - getTagReq = GetTagReq(spaceId, tagName, version) - getTagResp = self.client.getTag(getTagReq) + def get_tag(self, space_name, tag_name, version=-1): + space_id = self.get_space_id_from_cache(space_name) + get_tag_req = GetTagReq(space_id, tag_name, version) + get_tag_resp = self._client.getTag(get_tag_req) - if getTagResp.code == ErrorCode.SUCCEEDED: - return getTagResp.schema + if get_tag_resp.code == ErrorCode.SUCCEEDED: + return get_tag_resp.schema else: return None - def getTagSchema(self, spaceName, tagName, version=-1): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_tag_schema(self, space_name, tag_name, version=-1): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - getTagReq = GetTagReq(spaceId, tagName, version) - getTagResp = self.client.getTag(getTagReq) + get_tag_req = GetTagReq(space_id, tag_name, version) + get_tag_resp = self._client.getTag(get_tag_req) result = {} - for columnDef in getTagResp.schema.columns: - result[columnDef.name] = columnDef.type.type + for column_def in get_tag_resp.schema.columns: + result[column_def.name] = column_def.type.type return result - def getEdgeItemFromCache(self, spaceName, edgeName): - if spaceName not in self.spaceEdgeItems.keys(): + def get_edge_item_from_cache(self, space_name, edge_name): + if space_name not in self._space_edge_items.keys(): edges = {} - for edgeItem in self.getEdges(spaceName): - edges[edgeItem.edge_name] = edgeItem - self.spaceEdgeItems[spaceName] = edges + for edge_item in self.getEdges(space_name): + edges[edge_item.edge_name] = edge_item + self._space_edge_items[space_name] = edges - edgeItems = self.spaceEdgeItems[spaceName] - if edgeName in edgeItems.keys(): - return edgeItems[edgeName] + edge_items = self._space_edge_items[space_name] + if edge_name in edge_items.keys(): + return edge_items[edge_name] else: return None - def getEdgeNameFromCache(self, spaceName, edgeType): - if spaceName in self.edgeNameMap.keys(): - edgeNames = self.edgeNameMap[spaceName] - if edgeType in edgeNames.keys(): - return edgeNames[edgeType] + def get_edge_name_from_cache(self, space_name, edge_type): + if space_name in self._edge_name_map.keys(): + edge_names = self._edge_name_map[space_name] + if edge_type in edge_names.keys(): + return edge_names[edge_type] return None - def getEdges(self, spaceName): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_edges(self, space_name): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - listEdgesReq = ListEdgesReq(spaceId) - listEdgesResp =self.client.listEdges(listEdgesReq) - if listEdgesResp.code == ErrorCode.SUCCEEDED: - return listEdgesResp.edges + list_edges_req = ListEdgesReq(space_id) + list_edges_resp =self._client.listEdges(list_edges_req) + if list_edges_resp.code == ErrorCode.SUCCEEDED: + return list_edges_resp.edges else: - print('get tags error, error code: ', listEdgesResp.code) + print('get tags error, error code: ', list_edges_resp.code) return None - def getEdge(self, spaceName, edgeName, version=-1): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_edge(self, space_name, edge_name, version=-1): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - getEdgeReq = GetEdgeReq(spaceId, edgeName, version) - getEdgeResp = self.client.getEdge(getEdgeReq) - if getEdgeResp.code == ErrorCode.SUCCEEDED: - return getEdgeResp.Schema + get_edge_req = GetEdgeReq(space_id, edge_name, version) + get_edge_resp = self._client.getEdge(get_edge_req) + if get_edge_resp.code == ErrorCode.SUCCEEDED: + return get_edge_resp.Schema else: - print('get edge error, error code: ', getEdgeResp.code) + print('get edge error, error code: ', get_edge_resp.code) return None - def getEdgeSchema(self, spaceName, edgeName, version=-1): - spaceId = self.getSpaceIdFromCache(spaceName) - if spaceId == -1: + def get_edge_schema(self, space_name, edge_name, version=-1): + space_id = self.get_space_id_from_cache(space_name) + if space_id == -1: return None - getEdgeReq = GetEdgeReq(spaceId, edgeName, version) - getEdgeResp = self.client.getEdge(getEdgeReq) + get_edge_req = GetEdgeReq(space_id, edge_name, version) + get_edge_resp = self._client.getEdge(get_edge_req) result = {} - for columnDef in getEdgeResp.schema.columns: - result[columnDef.name] = columnDef.type.type + for column_def in get_edge_resp.schema.columns: + result[column_def.name] = column_def.type.type return result diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index 3d25a8f5..1f2f3148 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -23,7 +23,7 @@ class Iterator: def __init__(self, it=None): - self.it = iter(it) + self._it = iter(it) self._hasnext = None def __iter__(self): @@ -33,14 +33,14 @@ def next(self): if self._hasnext: result = self._thenext else: - result = next(self.it) + result = next(self._it) self._hasnext = None return result - def hasNext(self): + def has_next(self): if self._hasnext is None: try: - self._thenext = next(self.it) + self._thenext = next(self._it) except StopIteration: self._hasnext = False else: @@ -49,268 +49,270 @@ def hasNext(self): class ScanEdgeResponseIter: - def __init__(self, clientDad, space, leader, scanEdgeRequest, client): - self.clientDad = clientDad - self.space = space - self.leader = leader - self.scanEdgeRequest = scanEdgeRequest - self.client = client - self.cursor = None - self.haveNext = True - - def hasNext(self): - return self.haveNext + def __init__(self, client_dad, space, leader, scan_edge_request, client): + self._client_dad = client_dad + self._space = space + self._leader = leader + self._scan_edge_request = scan_edge_request + self._client = client + self._cursor = None + self._have_next = True + + def has_next(self): + return self._have_next def next(self): - self.scanEdgeRequest.cursor = self.cursor - scanEdgeResponse = self.client.scanEdge(self.scanEdgeRequest) - if scanEdgeResponse is None: - raise Exception('scanEdgeResponse is None') - self.cursor = scanEdgeResponse.next_cursor - self.haveNext = scanEdgeResponse.has_next - - if not self.clientDad.isSuccessfully(scanEdgeResponse): - self.leader, self.client = self.clientDad.handleResultCodes(scanEdgeResponse.result.failed_codes, self.space) - self.haveNext = False + self._scan_edge_request.cursor = self._cursor + scan_edge_response = self._client.scanEdge(self._scan_edge_request) + if scan_edge_response is None: + raise Exception('scan_edge_response is None') + self._cursor = scan_edge_response.next_cursor + self._have_next = scan_edge_response.has_next + + if not self._client_dad.is_successfully(scan_edge_response): + self._leader, self._client = self._client_dad.handle_result_codes(scan_edge_response.result.failed_codes, self._space) + self._haveNext = False return None else: - return scanEdgeResponse + return scan_edge_response return None class ScanVertexResponseIter: - def __init__(self, clientDad, space, leader, scanVertexRequest, client): - self.clientDad = clientDad - self.space = space - self.leader = leader - self.scanVertexRequest = scanVertexRequest - self.client = client - self.cursor = None - self.haveNext = True - - def hasNext(self): - return self.haveNext + def __init__(self, client_dad, space, leader, scan_vertex_request, client): + self._client_dad = client_dad + self._space = space + self._leader = leader + self._scan_vertex_request = scan_vertex_request + self._client = client + self._cursor = None + self._have_next = True + + def has_next(self): + return self._haveNext def next(self): - self.scanVertexRequest.cursor = self.cursor - scanVertexResponse = self.client.scanVertex(self.scanVertexRequest) - if scanVertexResponse is None: - raise Exception('scanVertexReponse is None') - self.cursor = scanVertexResponse.next_cursor - self.haveNext = scanVertexResponse.has_next - - if not self.clientDad.isSuccessfully(scanVertexResponse): - print('scanVertexResponse is not successfully, failed_codes: ', scanVertexResponse.result.failed_codes) - self.leader, self.client = self.clientDad.handleResultCodes(scanVertexResponse.result.failed_codes, self.space) - self.haveNext = False + self._scan_vertex_request.cursor = self._cursor + scan_vertex_response = self._client.scanVertex(self._scan_vertex_request) + if scan_vertex_response is None: + raise Exception('scan_vertex_reponse is None') + self._cursor = scan_vertex_response.next_cursor + self._haveNext = scan_vertex_response.has_next + + if not self._client_dad.is_successfully(scan_vertex_response): + print('scan_vertex_response is not successfully, failed_codes: ', scan_vertex_response.result.failed_codes) + self._leader, self._client = self._client_dad.handle_result_codes(scan_vertex_response.result.failed_codes, self._space) + self._have_next = False return None else: - return scanVertexResponse + return scan_vertex_response return None class ScanSpaceEdgeResponseIter: - def __init__(self, clientDad, space, partIdsIter, returnCols, allCols, limit, startTime, endTime): - self.clientDad = clientDad - self.scanEdgeResponseIter = None - self.space = space - self.partIdsIter = partIdsIter - self.returnCols = returnCols - self.allCols = allCols - self.limit = limit - self.startTime = startTime - self.endTime = endTime - - def hasNext(self): - return self.partIdsIter.hasNext() or self.scanEdgeResponseIter.hasNext() + def __init__(self, client_dad, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time): + self._client_dad = client_dad + self._scan_edge_response_iter = None + self._space = space + self._part_ids_iter = part_ids_iter + self._return_cols = return_cols + self._all_cols = all_cols + self._limit = limit + self._start_time = start_time + self._end_time = end_time + + def has_next(self): + return self._part_ids_iter.has_next() or self._scan_edge_response_iter.has_next() def next(self): - if self.scanEdgeResponseIter is None or not self.scanEdgeResponseIter.hasNext(): - part = self.partIdsIter.next() + if self._scan_edge_response_iter is None or not self._scan_edge_response_iter.has_next(): + part = self._part_ids_iter.next() if part is None: return None - leader = self.clientDad.getLeader(self.space, part) + leader = self._client_dad.get_leader(self._space, part) if leader is None: - raise Exception('part %s not found in space %s' % (part, self.space)) - spaceId = self.clientDad.metaClient.getSpaceIdFromCache(self.space) - if spaceId == -1: - raise Exception('space %s not found' % self.space) - colums = self.clientDad.getEdgeReturnCols(self.space, self.returnCols) - scanEdgeRequest = ScanEdgeRequest(spaceId, part, None, colums, self.allCols, self.limit, self.startTime, self.endTime) - self.scanEdgeResponseIter = self.clientDad.doScanEdge(self.space, leader, scanEdgeRequest) - assert(self.scanEdgeResponseIter is not None), 'scanEdgeResponseIter is None' + raise Exception('part %s not found in space %s' % (part, self._space)) + space_id = self._client_dad._meta_client.get_space_id_from_cache(self._space) + if space_id == -1: + raise Exception('space %s not found' % self._space) + colums = self._client_dad.get_edge_return_cols(self._space, self._return_cols) + scan_edge_request = ScanEdgeRequest(space_id, part, None, colums, self._all_cols, self._limit, self._start_time, self._end_time) + self._scan_edge_response_iter = self._client_dad.do_scan_edge(self._space, leader, scan_edge_request) + if self._scan_edge_response_iter is None: + raise Exception('scan_edge_response_iter is None') - return self.scanEdgeResponseIter.next() + return self._scan_edge_response_iter.next() class ScanSpaceVertexResponseIter: - def __init__(self, clientDad, space, partIdsIter, returnCols, allCols, limit, startTime, endTime): - self.clientDad = clientDad - self.scanVertexResponseIter = None - self.space = space - self.partIdsIter = partIdsIter - self.returnCols = returnCols - self.allCols = allCols - self.limit = limit - self.startTime = startTime - self.endTime = endTime - - def hasNext(self): - return self.partIdsIter.hasNext() or self.scanVertexResponseIter.hasNext() + def __init__(self, client_dad, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time): + self._client_dad = client_dad + self._scan_vertex_response_iter = None + self._space = space + self._part_ids_iter = part_ids_iter + self._return_cols = return_cols + self._all_cols = all_cols + self._limit = limit + self._start_time = start_time + self._end_time = end_time + + def has_next(self): + return self._part_ids_iter.has_next() or self._scan_vertex_response_iter.has_next() def next(self): - if self.scanVertexResponseIter is None or not self.scanVertexResponseIter.hasNext(): - part = self.partIdsIter.next() + if self._scan_vertex_response_iter is None or not self._scan_vertex_response_iter.has_next(): + part = self._part_ids_iter.next() if part is None: return None - leader = self.clientDad.getLeader(self.space, part) + leader = self._client_dad.get_leader(self._space, part) if leader is None: - raise Exception('part %s not found in space %s' % (part, self.space)) - spaceId = self.clientDad.metaClient.getSpaceIdFromCache(self.space) - if spaceId == -1: - raise Exception('space %s not found' % self.space) - colums = self.clientDad.getVertexReturnCols(self.space, self.returnCols) - scanVertexRequest = ScanVertexRequest(spaceId, part, None, colums, self.allCols, self.limit, self.startTime, self.endTime) - self.scanVertexResponseIter = self.clientDad.doScanVertex(self.space, leader, scanVertexRequest) - assert(self.scanVertexResponseIter is not None), 'scanVertexResponseIter is None' + raise Exception('part %s not found in space %s' % (part, self._space)) + space_id = self._client_dad._meta_client.get_space_id_from_cache(self._space) + if space_id == -1: + raise Exception('space %s not found' % self._space) + colums = self._client_dad.get_vertex_return_cols(self._space, self._return_cols) + scan_vertex_request = ScanVertexRequest(space_id, part, None, colums, self._all_cols, self._limit, self._start_time, self._end_time) + self._scan_vertex_response_iter = self._client_dad.do_scan_vertex(self._space, leader, scan_vertex_request) + if self._scan_vertex_response_iter is None: + raise Exception('scan_vertex_response_iter is None') - return self.scanVertexResponseIter.next() + return self._scan_vertex_response_iter.next() class StorageClient: - def __init__(self, metaClient): - self.metaClient = metaClient - self.clients = {} - self.leaders = {} - self.timeout = 1000 + def __init__(self, meta_client): + self._meta_client = meta_client + self._clients = {} + self._leaders = {} + self._timeout = 1000 def connect(self, address): - if address not in self.clients.keys(): - client = self.doConnect(address) - self.clients[address] = client + if address not in self._clients.keys(): + client = self.do_connect(address) + self._clients[address] = client return client else: - return self.clients[address] + return self._clients[address] - def disConnect(self, address): - self.clients.remove(address) + def disconnect(self, address): + self._clients.remove(address) - def doConnects(self, addresses): + def do_connects(self, addresses): for address in addresses: - client = self.doConnect(address) - self.clients[address] = client + client = self.do_connect(address) + self._clients[address] = client - def doConnect(self, address): + def do_connect(self, address): host = address[0] port = address[1] - tTransport = TSocket.TSocket(host, port) - tTransport.setTimeout(self.timeout) - tTransport = TTransport.TBufferedTransport(tTransport) - tProtocol = TBinaryProtocol.TBinaryProtocol(tTransport) - tTransport.open() - return Client(tProtocol) - - def scanEdge(self, space, returnCols, allCols, limit, startTime, endTime): - partIds = self.metaClient.getPartsAllocFromCache()[space].keys() - partIdsIter = Iterator(partIds) - return ScanSpaceEdgeResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) - - def scanPartEdge(self, space, part, returnCols, allCols, limit, startTime, endTime): - spaceId = self.metaClient.getSpaceIdFromCache(space) - columns = self.getEdgeReturnCols(space, returnCols) - scanEdgeRequest = ScanEdgeRequest(spaceId, part, None, columns, allCols, limit, startTime, endTime) - leader = self.getLeader(space, part) + transport = TSocket.TSocket(host, port) + transport.setTimeout(self._timeout) + transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + transport.open() + return Client(protocol) + + def scan_edge(self, space, return_cols, all_cols, limit, start_time, end_time): + part_ids = self._meta_client.get_parts_alloc_from_cache()[space].keys() + part_ids_iter = Iterator(part_ids) + return ScanSpaceEdgeResponseIter(self, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time) + + def scan_part_edge(self, space, part, return_cols, all_cols, limit, start_time, end_time): + space_id = self._meta_client.get_space_id_from_cache(space) + columns = self.get_edge_return_cols(space, return_cols) + scan_edge_request = ScanEdgeRequest(space_id, part, None, columns, all_cols, limit, start_time, end_time) + leader = self.get_leader(space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, space)) - return self.doScanEdge(space, leader, scanEdgeRequest) - - def scanVertex(self, space, returnCols, allCols, limit, startTime, endTime): - partIds = self.metaClient.getPartsAllocFromCache()[space].keys() - partIdsIter = Iterator(partIds) - return ScanSpaceVertexResponseIter(self, space, partIdsIter, returnCols, allCols, limit, startTime, endTime) - - def scanPartVertex(self, space, part, returnCols, allCols, limit, startTime, endTime): - spaceId = self.metaClient.getSpaceIdFromCache(space) - columns = self.getVertexReturnCols(space, returnCols) - scanVertexRequest = ScanVertexRequest(spaceId, part, None, columns, allCols, limit, startTime, endTime) - leader = self.getLeader(space, part) + return self.do_scan_edge(space, leader, scan_edge_request) + + def scan_vertex(self, space, return_cols, all_cols, limit, start_time, end_time): + part_ids = self._meta_client.get_parts_alloc_from_cache()[space].keys() + part_ids_iter = Iterator(part_ids) + return ScanSpaceVertexResponseIter(self, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time) + + def scan_part_vertex(self, space, part, return_cols, all_cols, limit, start_time, end_time): + space_id = self._meta_client.get_space_id_from_cache(space) + columns = self.get_vertex_return_cols(space, return_cols) + scan_vertex_request = ScanVertexRequest(space_id, part, None, columns, all_cols, limit, start_time, end_time) + leader = self.get_leader(space, part) if leader is None: raise Exception('part %s not found in space %s' % (part, space)) - return self.doScanVertex(space, leader, scanVertexRequest) + return self.do_scan_vertex(space, leader, scan_vertex_request) - def doScanEdge(self, space, leader, scanEdgeRequest): + def do_scan_edge(self, space, leader, scan_edge_request): client = self.connect(leader) if client is None: print('cannot connect to leader:', leader) - self.disConnect(leader) + self.disconnect(leader) return None - return ScanEdgeResponseIter(self, space, leader, scanEdgeRequest, client) + return ScanEdgeResponseIter(self, space, leader, scan_edge_request, client) - def doScanVertex(self, space, leader, scanVertexRequest): + def do_scan_vertex(self, space, leader, scan_vertex_request): client = self.connect(leader) if client is None: print('cannot connect to leader:', leader) - self.disConnect(leader) + self.disconnect(leader) return None - return ScanVertexResponseIter(self, space, leader, scanVertexRequest, client) + return ScanVertexResponseIter(self, space, leader, scan_vertex_request, client) - def getEdgeReturnCols(self, space, returnCols): + def get_edge_return_cols(self, space, return_cols): columns = {} - for edgeName, propNames in returnCols.items(): - edgeItem = self.metaClient.getEdgeItemFromCache(space, edgeName) - if edgeItem is None: - raise Exception('edge %s not found in space %s' % (edgeName, space)) - edgeType = edgeItem.edge_type - entryId = EntryId(edge_type=edgeType) - propDefs = [] - for propName in propNames: - propDef = PropDef(PropOwner.EDGE, entryId, propName) - propDefs.append(propDef) - columns[edgeType] = propDefs + for edge_name, prop_names in return_cols.items(): + edge_item = self._meta_client.get_edge_item_from_cache(space, edge_name) + if edge_item is None: + raise Exception('edge %s not found in space %s' % (edge_name, space)) + edge_type = edge_item.edge_type + entry_id = EntryId(edge_type=edge_type) + prop_defs = [] + for prop_name in prop_names: + prop_def = PropDef(PropOwner.EDGE, entry_id, prop_name) + prop_defs.append(prop_def) + columns[edge_type] = prop_defs return columns - def getVertexReturnCols(self, space, returnCols): + def get_vertex_return_cols(self, space, return_cols): columns = {} - for tagName, propNames in returnCols.items(): - tagItem = self.metaClient.getTagItemFromCache(space, tagName) - if tagItem is None: - raise Exception('tag %s not found in space %s' % (tagName, space)) - tagId = tagItem.tag_id - entryId = EntryId(tag_id=tagId) - propDefs = [] - for propName in propNames: - propDef = PropDef(PropOwner.SOURCE, entryId, propName) - propDefs.append(propDef) - columns[tagId] = propDefs + for tag_name, prop_names in return_cols.items(): + tag_item = self._meta_client.get_tag_item_from_cache(space, tag_name) + if tag_item is None: + raise Exception('tag %s not found in space %s' % (tag_name, space)) + tag_id = tag_item.tag_id + entry_id = EntryId(tag_id=tag_id) + prop_defs = [] + for prop_name in prop_names: + prop_def = PropDef(PropOwner.SOURCE, entry_id, prop_name) + prop_defs.append(prop_def) + columns[tag_id] = prop_defs return columns - def getLeader(self, spaceName, part): - return self.metaClient.getSpacePartLeaderFromCache(spaceName, part) - - def updateLeader(self, spaceName, partId, leader): - self.metaClient.updateSpacePartLeader(spaceName, partId, leader) - - def handleResultCodes(self, failedCodes, space): - for resultCode in failedCodes: - if resultCode.code == ErrorCode.E_LEADER_CHANGED: - print('ErrorCode.E_LEADER_CHANGED, leader changed to :', resultCode.leader) - hostAddr = resultCode.leader - if hostAddr is not None and hostAddr.ip != 0 and hostAddr.port != 0: - host = socket.inet_ntoa(struct.pack('I',socket.htonl(hostAddr.ip & 0xffffffff))) - port = hostAddr.port - newLeader = (host, port) - self.updateLeader(space, resultCode.part_id, newLeader) - if newLeader in self.clients.keys(): - newClient = self.clients[newLeader] + def get_leader(self, space_name, part): + return self._meta_client.get_space_part_leader_from_cache(space_name, part) + + def update_leader(self, space_name, part_id, leader): + self._meta_client.update_space_part_leader(space_name, part_id, leader) + + def handle_result_codes(self, failed_codes, space): + for result_code in failed_codes: + if result_code.code == ErrorCode.E_LEADER_CHANGED: + print('ErrorCode.E_LEADER_CHANGED, leader changed to :', result_code.leader) + host_addr = result_code.leader + if host_addr is not None and host_addr.ip != 0 and host_addr.port != 0: + host = socket.inet_ntoa(struct.pack('I',socket.htonl(host_addr.ip & 0xffffffff))) + port = host_addr.port + new_leader = (host, port) + self.update_leader(space, result_code.part_id, new_leader) + if new_leader in self._clients.keys(): + new_client = self._clients[new_leader] else: - newClient = None - return newLeader, newClient + new_client = None + return new_leader, new_client return None, None - def isSuccessfully(self, response): + def is_successfully(self, response): return len(response.result.failed_codes) == 0 diff --git a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py index 5843ee59..1ce1d3b3 100644 --- a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py @@ -13,37 +13,37 @@ from nebula.ngData.data import Result class ScanEdgeProcessor: - def __init__(self, metaClient): - self.metaClient = metaClient + def __init__(self, meta_client): + self._meta_client = meta_client - def process(self, spaceName, scanEdgeResponse): - rowReaders = {} + def process(self, space_name, scan_edge_response): + row_readers = {} rows = {} - edgeTypeNameMap = {} - - if scanEdgeResponse.edge_schema is not None: - for edgeType, schema in scanEdgeResponse.edge_schema.items(): - edgeName = self.metaClient.getEdgeNameFromCache(spaceName, edgeType) - edgeItem = self.metaClient.getEdgeItemFromCache(spaceName, edgeName) - schemaVersion = edgeItem.version - rowReaders[edgeType] = RowReader(schema, schemaVersion) - rows[edgeName] = [] - edgeTypeNameMap[edgeType] = edgeName + edge_type_name_map = {} + + if scan_edge_response.edge_schema is not None: + for edge_type, schema in scan_edge_response.edge_schema.items(): + edge_name = self._meta_client.get_edge_name_from_cache(space_name, edge_type) + edge_item = self._meta_client.get_edge_item_from_cache(space_name, edge_name) + schema_version = edge_item.version + row_readers[edge_type] = RowReader(schema, schema_version) + rows[edge_name] = [] + edge_type_name_map[edge_type] = edge_name else: - print('scanEdgeResponse.edge_schema is None') + print('scan_edge_response.edge_schema is None') - if scanEdgeResponse.edge_data is not None: - for scanEdge in scanEdgeResponse.edge_data: - edgeType = scanEdge.type - if edgeType not in rowReaders.keys(): + if scan_edge_response.edge_data is not None: + for scan_edge in scan_edge_response.edge_data: + edge_type = scan_edge.type + if edge_type not in row_readers.keys(): continue - rowReader = rowReaders[edgeType] - defaultProperties = rowReader.edgeKey(scanEdge.src, scanEdge.type, scanEdge.dst) - properties = rowReader.decodeValue(scanEdge.value) - edgeName = edgeTypeNameMap[edgeType] - rows[edgeName].append(Row(defaultProperties, properties)) + row_reader = row_readers[edge_type] + default_properties = row_reader.edge_key(scan_edge.src, scan_edge.type, scan_edge.dst) + properties = row_reader.decode_value(scan_edge.value) + edge_name = edge_type_name_map[edge_type] + rows[edge_name].append(Row(default_properties, properties)) else: - print('scanEdgeResponse.edge_data is None') + print('scan_edge_response.edge_data is None') return Result(rows) diff --git a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py index 90612dfa..e05c09ea 100644 --- a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py @@ -13,39 +13,39 @@ from nebula.ngData.data import Result class ScanVertexProcessor: - def __init__(self, metaClient): - self.metaClient = metaClient + def __init__(self, meta_client): + self._meta_client = meta_client - def process(self, spaceName, scanVertexResponse): - if scanVertexResponse is None: - print('process: scanVertexResponse is None') + def process(self, space_name, scan_vertex_response): + if scan_vertex_response is None: + print('process: scan_vertex_response is None') return None - rowReaders = {} + row_readers = {} rows = {} - tagIdNameMap = {} - if scanVertexResponse.vertex_schema is not None: - for tagId, schema in scanVertexResponse.vertex_schema.items(): - tagName = self.metaClient.getTagNameFromCache(spaceName, tagId) - tagItem = self.metaClient.getTagItemFromCache(spaceName, tagName) - schemaVersion = tagItem.version - rowReaders[tagId] = RowReader(schema, schemaVersion) - rows[tagName] = [] - tagIdNameMap[tagId] = tagName + tag_id_name_map = {} + if scan_vertex_response.vertex_schema is not None: + for tag_id, schema in scan_vertex_response.vertex_schema.items(): + tag_name = self._meta_client.get_tag_name_from_cache(space_name, tag_id) + tag_item = self._meta_client.get_tag_item_from_cache(space_name, tag_name) + schema_version = tag_item.version + row_readers[tag_id] = RowReader(schema, schema_version) + rows[tag_name] = [] + tag_id_name_map[tag_id] = tag_name else: - print('scanVertexResponse.vertex_schema is None') + print('scan_vertex_response.vertex_schema is None') - if scanVertexResponse.vertex_data is not None: - for scanTag in scanVertexResponse.vertex_data: - tagId = scanTag.tagId - if tagId not in rowReaders.keys(): + if scan_vertex_response.vertex_data is not None: + for scan_tag in scan_vertex_response.vertex_data: + tag_id = scan_tag.tagId + if tag_id not in row_readers.keys(): continue - rowReader = rowReaders[tagId] - defaultProperties = rowReader.vertexKey(scanTag.vertexId, scanTag.tagId) - properties = rowReader.decodeValue(scanTag.value) - tagName = tagIdNameMap[tagId] - rows[tagName].append(Row(defaultProperties, properties)) + row_reader = row_readers[tag_id] + default_properties = row_reader.vertex_key(scan_tag.vertexId, scan_tag.tagId) + properties = row_reader.decode_value(scan_tag.value) + tag_name = tag_id_name_map[tag_id] + rows[tag_name].append(Row(default_properties, properties)) else: - print('scanVertexResponse.vertex_data is None') + print('scan_vertex_response.vertex_data is None') return Result(rows) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 9a79b2c0..2a25c8e3 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -7,7 +7,7 @@ """ -Nebula StorageClient tests. +Nebula storage_client tests. """ import pytest @@ -40,12 +40,12 @@ def test_prepare(): return resp = client.authenticate('user', 'password') assert resp.error_code == 0, resp.error_msg - resp = client.execute('DROP SPACE IF EXISTS %s' % spaceName) + resp = client.execute('DROP SPACE IF EXISTS %s' % space_name) assert resp.error_code == 0, resp.error_msg - resp = client.execute('CREATE SPACE %s(partition_num=1)' % spaceName) + resp = client.execute('CREATE SPACE %s(partition_num=1)' % space_name) assert resp.error_code == 0, resp.error_msg time.sleep(5) - resp = client.execute('USE %s' % spaceName) + resp = client.execute('USE %s' % space_name) assert resp.error_code == 0, resp.error_msg time.sleep(5) resp = client.execute('CREATE TAG player(name string, age int)') @@ -64,51 +64,51 @@ def test_prepare(): client.sign_out() assert False -def test_scanEdge(): - result = storageClient.scanEdge(spaceName, {'follow':['degree']}, True, 100, 0, sys.maxsize) +def test_scan_edge(): + result = storage_client.scan_edge(space_name, {'follow':['degree']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().edge_data is not None -def test_scanVertex(): - result = storageClient.scanVertex(spaceName, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) +def test_scan_vertex(): + result = storage_client.scan_vertex(space_name, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().vertex_data is not None -def test_scanPartEdge(): - result = storageClient.scanPartEdge(spaceName, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) +def test_scan_part_edge(): + result = storage_client.scan_part_edge(space_name, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().edge_data is not None -def test_scanPartVertex(): - result = storageClient.scanPartVertex(spaceName, 1, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) +def test_scan_part_vertex(): + result = storage_client.scan_part_vertex(space_name, 1, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().vertex_data is not None -def test_getTagSchema(): - result = metaClient.getTagSchema(spaceName, 'player') +def test_get_tag_schema(): + result = meta_client.get_tag_schema(space_name, 'player') expect = {'name':6, 'age':2} assert result == expect -def test_getEdgeSchema(): - result = metaClient.getEdgeSchema(spaceName, 'follow') +def test_get_edge_schema(): + result = meta_client.get_edge_schema(space_name, 'follow') expect = {'degree': 5} assert result == expect -def test_getEdgeReturnCols(): - edgeItem = metaClient.getEdgeItemFromCache(spaceName, 'follow') - edgeType = edgeItem.edge_type - entryId = EntryId(edge_type=edgeType) - result = storageClient.getEdgeReturnCols(spaceName, {'follow':['degree']}) - expect = {edgeType:[PropDef(PropOwner.EDGE, entryId, 'degree')]} +def test_get_edge_return_cols(): + edge_item = meta_client.get_edge_item_from_cache(space_name, 'follow') + edge_type = edge_item.edge_type + entry_id = EntryId(edge_type=edge_type) + result = storage_client.get_edge_return_cols(space_name, {'follow':['degree']}) + expect = {edge_type:[PropDef(PropOwner.EDGE, entry_id, 'degree')]} assert result == expect -def test_getVertexReturnCols(): - tagItem = metaClient.getTagItemFromCache(spaceName, 'player') - tagId = tagItem.tag_id - entryId = EntryId(tag_id=tagId) - result = storageClient.getVertexReturnCols(spaceName, {'player':['name', 'age']}) - expect = {tagId:[PropDef(PropOwner.SOURCE, entryId, 'name'), PropDef(PropOwner.SOURCE, entryId, 'age')]} +def test_get_vertex_return_cols(): + tag_item = meta_client.get_tag_item_from_cache(space_name, 'player') + tag_id = tag_item.tag_id + entry_id = EntryId(tag_id=tag_id) + result = storage_client.get_vertex_return_cols(space_name, {'player':['name', 'age']}) + expect = {tag_id:[PropDef(PropOwner.SOURCE, entry_id, 'name'), PropDef(PropOwner.SOURCE, entry_id, 'age')]} assert result == expect -def test_handleResultCodes(): - failedCodes = [ResultCode(code=ErrorCode.E_LEADER_CHANGED, part_id=1, leader=HostAddr(ip=2130706433, port=storage_port))] - result, _ = storageClient.handleResultCodes(failedCodes, spaceName) +def test_handle_result_codes(): + failed_codes = [ResultCode(code=ErrorCode.E_LEADER_CHANGED, part_id=1, leader=HostAddr(ip=2130706433, port=storage_port))] + result, _ = storage_client.handle_result_codes(failed_codes, space_name) expect = (host, storage_port) assert result == expect @@ -117,7 +117,7 @@ def test_handleResultCodes(): meta_port = 45500 graph_port = 3699 storage_port = 44500 -spaceName = 'test_storage' -metaClient = MetaClient([(host, meta_port)]) -metaClient.connect() -storageClient = StorageClient(metaClient) +space_name = 'test_storage' +meta_client = MetaClient([(host, meta_port)]) +meta_client.connect() +storage_client = StorageClient(meta_client) From 6e1b8232ad51243a5c7291247bfd105eae6dc43d Mon Sep 17 00:00:00 2001 From: jievince Date: Tue, 25 Aug 2020 11:15:19 +0800 Subject: [PATCH 11/17] catch exception of thrift --- nebula/ngMeta/MetaClient.py | 21 +++++++++++++-------- nebula/ngStorage/StorageClient.py | 17 +++++++++++------ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/nebula/ngMeta/MetaClient.py b/nebula/ngMeta/MetaClient.py index 1d039249..2aeeb067 100644 --- a/nebula/ngMeta/MetaClient.py +++ b/nebula/ngMeta/MetaClient.py @@ -75,14 +75,19 @@ def do_connect(self, addresses): address = addresses[random.randint(0, len(addresses)-1)] host = address[0] port = address[1] - transport = TSocket.TSocket(host, port) - transport.setTimeout(self._timeout) - transport = TTransport.TBufferedTransport(transport) - protocol = TBinaryProtocol.TBinaryProtocol(transport) - transport.open() - self._client = Client(protocol) - self.update_schemas() - RepeatTimer(2, self.update_schemas).start() # call updatSchemas() every 2 seconds + try: + transport = TSocket.TSocket(host, port) + transport.setTimeout(self._timeout) + transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + transport.open() + except Exception as x: + print(x) + return -1 + else: + self._client = Client(protocol) + self.update_schemas() + RepeatTimer(2, self.update_schemas).start() # call updatSchemas() every 2 seconds return 0 diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index 1f2f3148..5bde67e1 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -207,12 +207,17 @@ def do_connects(self, addresses): def do_connect(self, address): host = address[0] port = address[1] - transport = TSocket.TSocket(host, port) - transport.setTimeout(self._timeout) - transport = TTransport.TBufferedTransport(transport) - protocol = TBinaryProtocol.TBinaryProtocol(transport) - transport.open() - return Client(protocol) + try: + transport = TSocket.TSocket(host, port) + transport.setTimeout(self._timeout) + transport = TTransport.TBufferedTransport(transport) + protocol = TBinaryProtocol.TBinaryProtocol(transport) + transport.open() + except Exception as x: + print(x) + return None + else: + return Client(protocol) def scan_edge(self, space, return_cols, all_cols, limit, start_time, end_time): part_ids = self._meta_client.get_parts_alloc_from_cache()[space].keys() From 1609ef007d2d04b1da79af703c1b1b02384e8fdf Mon Sep 17 00:00:00 2001 From: jievince Date: Fri, 28 Aug 2020 15:20:30 +0800 Subject: [PATCH 12/17] add annotation --- examples/StorageClientExample.py | 7 + nebula/ngData/data.py | 62 +++--- nebula/ngMeta/MetaClient.py | 183 +++++++++++++++--- nebula/ngStorage/StorageClient.py | 58 +++++- .../ngProcessor/ScanEdgeProcessor.py | 15 +- .../ngProcessor/ScanVertexProcessor.py | 15 +- tests/test_StorageClient.py | 31 +-- 7 files changed, 299 insertions(+), 72 deletions(-) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index 690670b2..5aa319d5 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -99,20 +99,27 @@ def get_return_cols(space): if __name__ == '__main__': + # initialize a MetaClient to establish connections with meta servers meta_client = MetaClient([(sys.argv[1], sys.argv[2])]) code = meta_client.connect() if code == ErrorCode.E_FAIL_TO_CONNECT: raise Exception('connect to %s:%d failed' % (sys.argv[1], sys.argv[2])) + # initialize a StorageClient storage_client = StorageClient(meta_client) + # initialize a ScanEdgeProcessor to process scanned edge data scan_edge_processor = ScanEdgeProcessor(meta_client) + # initialize a ScanVertexProcessor to process scanned vertex data scan_vertex_processor = ScanVertexProcessor(meta_client) space_to_read = sys.argv[3] + # get argument return_cols, which is used in function scan_edge, scan_vertex, scan_part_edge, scan_part_vertex vertex_return_cols, edge_return_cols = get_return_cols(space_to_read) all_cols = True if space_to_read not in meta_client.get_parts_alloc_from_cache().keys(): raise Exception('spaceToRead %s is not found in nebula' % space_to_read) else: + # scan vertex data scan_vertex(space_to_read, vertex_return_cols, all_cols) + # scan edge data scan_edge(space_to_read, edge_return_cols, all_cols) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 46367d1d..30e9a4f2 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -5,14 +5,11 @@ # This source code is licensed under Apache 2.0 License, # attached with Common Clause Condition 1.0, found in the LICENSES directory. -from enum import Enum +from common.ttypes import SupportedType import struct import six class PropertyDef: - PropertyType = Enum('PropertyType', ('UNKNOWN', 'BOOL', 'INT', 'VID', 'FLOAT', 'DOUBLE', \ - 'STRING', 'TIMESTAMP', 'VERTEX_ID', 'TAG_ID', 'SRC_ID', 'EDGE_TYPE', 'EDGE_RANK', 'DST_ID')) - def __init__(self, property_type, name): self._property_type = property_type self._name = name @@ -40,6 +37,12 @@ def __init__(self, default_properties, properties): class RowReader: def __init__(self, schema, schema_version=0): + """Initalizer + Arguments: + - schema: schema of tag or edge + - schema_version: version of schema + Returns: emtpy + """ self._schema_version = schema_version self._defs = [] self._field_num = 0 @@ -47,12 +50,9 @@ def __init__(self, schema, schema_version=0): self._property_types = [] self._offset = 0 - prop_type_map = {0: PropertyDef.PropertyType.UNKNOWN, 1: PropertyDef.PropertyType.BOOL, 2: PropertyDef.PropertyType.INT, - 3: PropertyDef.PropertyType.VID, 4: PropertyDef.PropertyType.FLOAT, 5: PropertyDef.PropertyType.DOUBLE, - 6: PropertyDef.PropertyType.STRING, 21: PropertyDef.PropertyType.TIMESTAMP} idx = 0 for column_def in schema.columns: - property_type = prop_type_map[column_def.type.type] # ColumnDef is in common/ttypes.py + property_type = column_def.type.type column_name = column_def.name self._defs.append((column_name, property_type)) self._property_name_index[column_name] = idx @@ -60,6 +60,12 @@ def __init__(self, schema, schema_version=0): self._field_num = len(self._defs) def decode_value(self, value, schema_version=None): + """ decode data + Arguments: + - value: data value scanned from the storage server + - schema_version: version of schema + Returns: decoded property values + """ if schema_version is None: schema_version = self._schema_version self._offset = 0 @@ -86,34 +92,34 @@ def decode_value(self, value, schema_version=None): for i in range(len(self._defs)): field = self._defs[i][0] property_type = self._defs[i][1] - if property_type == PropertyDef.PropertyType.BOOL: + if property_type == SupportedType.BOOL: properties.append(self.get_bool_property(field, value)) - elif property_type == PropertyDef.PropertyType.INT: + elif property_type == SupportedType.INT: properties.append(self.get_int_property(field, value)) - elif property_type == PropertyDef.PropertyType.FLOAT: #unused now + elif property_type == SupportedType.FLOAT: #unused now properties.append(self.get_float_property(field, value)) - elif property_type == PropertyDef.PropertyType.DOUBLE: + elif property_type == SupportedType.DOUBLE: properties.append(self.get_double_property(field, value)) - elif property_type == PropertyDef.PropertyType.STRING: + elif property_type == SupportedType.STRING: properties.append(self.get_string_property(field, value)) - elif property_type == PropertyDef.PropertyType.TIMESTAMP: + elif property_type == SupportedType.TIMESTAMP: properties.append(self.get_timestamp_property(field, value)) else: - raise Exception('Invalid propertyType in schema: ', property_type) + raise Exception('Unsupported propertyType in schema: ', property_type) return properties - def edge_key(self, srcId, edgeType, dstId): + def edge_key(self, srcId, edgeName, dstId): properties = [] - properties.append(Property(PropertyDef.PropertyType.SRC_ID, "_srcId", srcId)) - properties.append(Property(PropertyDef.PropertyType.EDGE_TYPE, "_edgeType", edgeType)) - properties.append(Property(PropertyDef.PropertyType.DST_ID, "_dstId", dstId)) + properties.append(Property(SupportedType.VID, "_src", srcId)) + properties.append(Property(SupportedType.STRING, "_edge", edgeName)) + properties.append(Property(SupportedType.VID, "_dst", dstId)) return properties - def vertex_key(self, vertexId, tagId): + def vertex_key(self, vertexId, tagName): properties = [] - properties.append(Property(PropertyDef.PropertyType.VERTEX_ID, "_vertexId", vertexId)) - properties.append(Property(PropertyDef.PropertyType.TAG_ID, "_tagId", tagId)) + properties.append(Property(SupportedType.VID, "_vid", vertexId)) + properties.append(Property(SupportedType.STRING, "_tag", tagName)) return properties def get_property(self, row, name): @@ -132,26 +138,26 @@ def get_bool_property(self, name, value): else: val = value[self._offset] != 0x00 self._offset += 1 - return Property(PropertyDef.PropertyType.BOOL, name, val) + return Property(SupportedType.BOOL, name, val) def get_int_property(self, name, value): val = self.read_compressed_int(value) - return Property(PropertyDef.PropertyType.INT, name, val) #### 字节流解析出data + return Property(SupportedType.INT, name, val) #### 字节流解析出data def get_timestamp_property(self, name, value): val = self.read_compressed_int(value) # val = datetime.fromtimestamp(val) - return Property(PropertyDef.PropertyType.TIMESTAMP, name, val) + return Property(SupportedType.TIMESTAMP, name, val) def get_float_property(self, name, value): val = struct.unpack_from(' self._space_part_location = {} # map>> self._space_part_leader = {} # map> @@ -64,6 +70,11 @@ def __init__(self, addresses, timeout=1000, self._client = None def connect(self): + """connect to meta servers + Arguments: emtpy + Returns: + - error_code: the code indicates whether the connection is successful + """ while self._connection_retry > 0: code = self.do_connect(self._addresses) if code == 0: @@ -81,15 +92,13 @@ def do_connect(self, addresses): transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(transport) transport.open() - except Exception as x: - print(x) - return -1 - else: self._client = Client(protocol) self.update_schemas() RepeatTimer(2, self.update_schemas).start() # call updatSchemas() every 2 seconds - - return 0 + return 0 + except Exception as x: + print(x) + return -1 def update_schemas(self): for space_id_name in self.list_spaces(): @@ -115,16 +124,30 @@ def update_schemas(self): edges_name[edge_item.edge_type] = edge_item.edge_name self._space_edge_items[space_name] = edges self._edge_name_map[space_name] = edges_name + + # Update leader of partions self.set_space_part_leader() - return 0 def get_space_id_from_cache(self, space_name): + """get space id of the space + Arguments: + - space_name: name of the space + Returns: + - space_id: id of the space + """ if space_name not in self._space_name_map.keys(): return -1 else: return self._space_name_map[space_name] def get_space_part_leader_from_cache(self, space_name, part_id): + """get leader of the partion + Arguments: + - space_name: name of the space + - part_id: id of the partition + Returns: + - leader address of the partition + """ if space_name not in self._space_part_leader.keys(): return None if part_id not in self._space_part_leader[space_name].keys(): @@ -150,21 +173,36 @@ def set_space_part_leader(self): self._space_part_leader[space][part_id] = leader def list_spaces(self): + """list all of the spaces + Arguments: empty + Returns: + - spaces: IdName of all spaces + IdName's attributes: + - id + - name + """ list_spaces_req = ListSpacesReq() list_spaces_resp = self._client.listSpaces(list_spaces_req) if list_spaces_resp.code == ErrorCode.SUCCEEDED: - return list_spaces_resp.spaces # space_nameID--> IdName + return list_spaces_resp.spaces # IdName else: print('list spaces error, error code: ', list_spaces_resp.code) return None - def get_part_from_cache(self, space_name, part): - if space_name not in self._space_part_location.keys(): - self._space_part_location[space_name] = self.get_parts_alloc(space_name) - parts_alloc = self._space_part_location[space_name] - if parts_alloc is None or part not in parts_alloc.keys(): - return None - return parts_alloc[part] + def get_part_alloc_from_cache(self, space_name, part_id): + """get addresses of the partition + Arguments: + - space_name: name of the space + - part_id: id of the partition + Returns: + - addresses: addresses of the partition + """ + if space_name in self._space_part_location.keys(): + parts_alloc = self._space_part_location[space_name] + if part_id in parts_alloc.keys(): + return parts_alloc[part] + + return None def get_parts_alloc(self, space_name): space_id = self.get_space_id_from_cache(space_name) @@ -189,23 +227,39 @@ def get_parts_alloc(self, space_name): return None def get_parts_alloc_from_cache(self): + """ get addresses of partitions of spaces + Arguments: empty + Returns: + - space_part_location: map>> + """ return self._space_part_location - def get_part_alloc_from_cache(self, space_name, part): - if space_name in self._space_part_location.keys(): - parts_alloc = self._space_part_location[space_name] - if part in parts_alloc.keys(): - return parts_alloc[part] - - return None - def get_tag_item_from_cache(self, space_name, tag_name): + """ get TagItem of the tag + Arguments: + - space_name: name of the space + - tag_name: name of the tag + Returns: + - TagItem + TagItem's attributes: + - tag_id + - tag_name + - version + - schema + """ if space_name in self._space_tag_items.keys() and tag_name in self._space_tag_items[space_name].keys(): return self._space_tag_items[space_name][tag_name] return None def get_tag_name_from_cache(self, space_name, tag_id): + """ get tag_name of the tag + Arguments: + - space_name: name of the space + - tag_id: id of the tag + Returns: + - tag_name: name of the tag + """ if space_name in self._tag_name_map.keys(): tag_names = self._tag_name_map[space_name] if tag_id in tag_names.keys(): @@ -214,6 +268,17 @@ def get_tag_name_from_cache(self, space_name, tag_id): return None def get_tags(self, space_name): + """ get TagItems of the space + Arguments: + - space_name: name of the space + Returns: + - tags: TagItems + TagItem's attributes: + - tag_id + - tag_name + - version + - schema + """ space_id = self.get_space_id_from_cache(space_name) if space_id == -1: return None @@ -227,6 +292,17 @@ def get_tags(self, space_name): return None def get_tag(self, space_name, tag_name, version=-1): + """ get tag schema of the given version + Arguments: + - space_name: name of the space + - tag_name: name of the tag + - version: version of the tag schema + Returns: + - Schema: tag schema of the given version + Schema's attributes: + - columns + - schema_prop + """ space_id = self.get_space_id_from_cache(space_name) get_tag_req = GetTagReq(space_id, tag_name, version) get_tag_resp = self._client.getTag(get_tag_req) @@ -237,6 +313,14 @@ def get_tag(self, space_name, tag_name, version=-1): return None def get_tag_schema(self, space_name, tag_name, version=-1): + """ get tag schema columns of the given version + Arguments: + - space_name: name of the space + - tag_name: name of the tag + - version: version of the tag schema + Returns: + - result: columns of the tag schema + """ space_id = self.get_space_id_from_cache(space_name) if space_id == -1: return None @@ -248,6 +332,18 @@ def get_tag_schema(self, space_name, tag_name, version=-1): return result def get_edge_item_from_cache(self, space_name, edge_name): + """ get EdgeItem of the edge + Arguments: + - space_name: name of the space + - edge_name: name of the edge + Returns: + - EdgeItem + EdgeItem's attributes: + - edge_type + - edge_name + - version + - schema + """ if space_name not in self._space_edge_items.keys(): edges = {} for edge_item in self.getEdges(space_name): @@ -261,6 +357,13 @@ def get_edge_item_from_cache(self, space_name, edge_name): return None def get_edge_name_from_cache(self, space_name, edge_type): + """ get edge name of the edge + Arguments: + - space_name: name of the space + - edge_type: edge type of the edge + Returns: + - edge_name: name of the edge + """ if space_name in self._edge_name_map.keys(): edge_names = self._edge_name_map[space_name] if edge_type in edge_names.keys(): @@ -269,6 +372,17 @@ def get_edge_name_from_cache(self, space_name, edge_type): return None def get_edges(self, space_name): + """ get EdgeItems of the space + Arguments: + - space_name: name of the space + Returns: + - edges: EdgeItems + EdgeItem's attributes: + - edge_type + - edge_name + - version + - schema + """ space_id = self.get_space_id_from_cache(space_name) if space_id == -1: return None @@ -281,6 +395,17 @@ def get_edges(self, space_name): return None def get_edge(self, space_name, edge_name, version=-1): + """ get edge schema of the given version + Arguments: + - space_name: name of the space + - edge_name: name of the edge + - version: version of the edge schema + Returns: + - schema of the edge + Schema's attributes: + - columns + - schema_prop + """ space_id = self.get_space_id_from_cache(space_name) if space_id == -1: return None @@ -293,6 +418,14 @@ def get_edge(self, space_name, edge_name, version=-1): return None def get_edge_schema(self, space_name, edge_name, version=-1): + """ get edge schema columns of the given version + Arguments: + - space_name: name of the space + - edge_name: name of the edge + - version: version of the edge schema + Returns: + - result: columns of the edge schema + """ space_id = self.get_space_id_from_cache(space_name) if space_id == -1: return None diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index 5bde67e1..efad5a44 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -183,12 +183,23 @@ def next(self): class StorageClient: def __init__(self, meta_client): + """Initializer + Arguments: + - meta_clent: an initialized MetaClient + Returns: empty + """ self._meta_client = meta_client self._clients = {} self._leaders = {} self._timeout = 1000 def connect(self, address): + """ connect to storage server + Arguments: + - address: the address of the storage servers + Returns: + - client: a storage client object + """ if address not in self._clients.keys(): client = self.do_connect(address) self._clients[address] = client @@ -213,18 +224,39 @@ def do_connect(self, address): transport = TTransport.TBufferedTransport(transport) protocol = TBinaryProtocol.TBinaryProtocol(transport) transport.open() + return Client(protocol) except Exception as x: print(x) return None - else: - return Client(protocol) def scan_edge(self, space, return_cols, all_cols, limit, start_time, end_time): + """ scan edges of a space + Arguments: + - space: name of the space to scan + - return_cols: the edge's attribute columns to be returned + - all_cols: whether to return all attribute columns. + - limit: maximum number of data returned + - start_time: start time of the edge data to return + - end_time: end time of the edge data to return + Returns: + - iter: an iterator that can traverse all scanned edge data + """ part_ids = self._meta_client.get_parts_alloc_from_cache()[space].keys() part_ids_iter = Iterator(part_ids) return ScanSpaceEdgeResponseIter(self, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time) def scan_part_edge(self, space, part, return_cols, all_cols, limit, start_time, end_time): + """ scan edges of a partition + Arguments: + - space: name of the space to scan + - return_cols: the edge's attribute columns to be returned + - all_cols: whether to return all attribute columns. + - limit: maximum number of data returned + - start_time: start time of the edge data to return + - end_time: end time of the edge data to return + Returns: + - iter: an iterator that can traverse all scanned edge data + """ space_id = self._meta_client.get_space_id_from_cache(space) columns = self.get_edge_return_cols(space, return_cols) scan_edge_request = ScanEdgeRequest(space_id, part, None, columns, all_cols, limit, start_time, end_time) @@ -234,11 +266,33 @@ def scan_part_edge(self, space, part, return_cols, all_cols, limit, start_time, return self.do_scan_edge(space, leader, scan_edge_request) def scan_vertex(self, space, return_cols, all_cols, limit, start_time, end_time): + """ scan vertexes of a space + Arguments: + - space: name of the space to scan + - return_cols: the tag's attribute columns to be returned + - all_cols: whether to return all attribute columns. + - limit: maximum number of data returned + - start_time: start time of the vertex data to return + - end_time: end time of the vertex data to return + Returns: + - iter: an iterator that can traverse all scanned vertex data + """ part_ids = self._meta_client.get_parts_alloc_from_cache()[space].keys() part_ids_iter = Iterator(part_ids) return ScanSpaceVertexResponseIter(self, space, part_ids_iter, return_cols, all_cols, limit, start_time, end_time) def scan_part_vertex(self, space, part, return_cols, all_cols, limit, start_time, end_time): + """ scan vertexes of a partition + Arguments: + - space: name of the space to scan + - return_cols: the tag's attribute columns to be returned + - all_cols: whether to return all attribute columns. + - limit: maximum number of data returned + - start_time: start time of the vertex data to return + - end_time: end time of the vertex data to return + Returns: + - iter: an iterator that can traverse all scanned vertex data + """ space_id = self._meta_client.get_space_id_from_cache(space) columns = self.get_vertex_return_cols(space, return_cols) scan_vertex_request = ScanVertexRequest(space_id, part, None, columns, all_cols, limit, start_time, end_time) diff --git a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py index 1ce1d3b3..a91ded35 100644 --- a/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanEdgeProcessor.py @@ -14,9 +14,21 @@ class ScanEdgeProcessor: def __init__(self, meta_client): + """Initializer + Arguments: + - meta_client: an initialized MetaClient + Returns: emtpy + """ self._meta_client = meta_client def process(self, space_name, scan_edge_response): + """ process the ScanEdgeResponse + Arguments: + - space_name: name of the space + - scan_edge_response: response of storage server + Returns: + - result: a dataset of edges and its property values + """ row_readers = {} rows = {} edge_type_name_map = {} @@ -39,7 +51,8 @@ def process(self, space_name, scan_edge_response): continue row_reader = row_readers[edge_type] - default_properties = row_reader.edge_key(scan_edge.src, scan_edge.type, scan_edge.dst) + edge_name = self._meta_client.get_edge_name_from_cache(space_name, edge_type) + default_properties = row_reader.edge_key(scan_edge.src, edge_name, scan_edge.dst) properties = row_reader.decode_value(scan_edge.value) edge_name = edge_type_name_map[edge_type] rows[edge_name].append(Row(default_properties, properties)) diff --git a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py index e05c09ea..1e158f24 100644 --- a/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py +++ b/nebula/ngStorage/ngProcessor/ScanVertexProcessor.py @@ -14,9 +14,21 @@ class ScanVertexProcessor: def __init__(self, meta_client): + """Initializer + Arguments: + - meta_client: an initialized MetaClient + Returns: emtpy + """ self._meta_client = meta_client def process(self, space_name, scan_vertex_response): + """ process the ScanVertexResponse + Arguments: + - space_name: name of the space + - scan_vertex_response: response of the storage server + Returns: + - result: a dataset of tags and its property values + """ if scan_vertex_response is None: print('process: scan_vertex_response is None') return None @@ -41,7 +53,8 @@ def process(self, space_name, scan_vertex_response): continue row_reader = row_readers[tag_id] - default_properties = row_reader.vertex_key(scan_tag.vertexId, scan_tag.tagId) + tag_name = self._meta_client.get_tag_name_from_cache(space_name, tag_id) + default_properties = row_reader.vertex_key(scan_tag.vertexId, tag_name) properties = row_reader.decode_value(scan_tag.value) tag_name = tag_id_name_map[tag_id] rows[tag_name].append(Row(default_properties, properties)) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 2a25c8e3..d1780825 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -23,6 +23,7 @@ from storage.ttypes import ResultCode from storage.ttypes import ErrorCode from common.ttypes import HostAddr +from common.ttypes import SupportedType from graph import ttypes from nebula.ConnectionPool import ConnectionPool from nebula.Client import GraphClient @@ -48,16 +49,16 @@ def test_prepare(): resp = client.execute('USE %s' % space_name) assert resp.error_code == 0, resp.error_msg time.sleep(5) - resp = client.execute('CREATE TAG player(name string, age int)') + resp = client.execute('CREATE TAG player(name string, age int, married bool)') assert resp.error_code == 0, resp.error_msg - resp = client.execute('CREATE EDGE follow(degree double)') + resp = client.execute('CREATE EDGE follow(degree double, time timestamp)') assert resp.error_code == 0, resp.error_msg time.sleep(12) - resp = client.execute('INSERT VERTEX player(name, age) VALUES 1:(\'Bob\', 18)') + resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 1:(\'Bob\', 18, FALSE)') assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT VERTEX player(name, age) VALUES 2:(\'Tome\', 22)') + resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 2:(\'Tome\', 22, TRUE)') assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT EDGE follow(degree) VALUES 1->2:(94.7)') + resp = client.execute('INSERT EDGE follow(degree, time) VALUES 1->2:(94.7, \'2010-09-01 08:00:00\')') assert resp.error_code == 0, resp.error_msg except Exception as ex: print(ex) @@ -65,45 +66,45 @@ def test_prepare(): assert False def test_scan_edge(): - result = storage_client.scan_edge(space_name, {'follow':['degree']}, True, 100, 0, sys.maxsize) + result = storage_client.scan_edge(space_name, {'follow':['degree', 'time']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().edge_data is not None def test_scan_vertex(): - result = storage_client.scan_vertex(space_name, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) + result = storage_client.scan_vertex(space_name, {'player':['name', 'age', 'married']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().vertex_data is not None def test_scan_part_edge(): - result = storage_client.scan_part_edge(space_name, 1, {'follow':['degree']}, True, 100, 0, sys.maxsize) + result = storage_client.scan_part_edge(space_name, 1, {'follow':['degree', 'time']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().edge_data is not None def test_scan_part_vertex(): - result = storage_client.scan_part_vertex(space_name, 1, {'player':['name', 'age']}, True, 100, 0, sys.maxsize) + result = storage_client.scan_part_vertex(space_name, 1, {'player':['name', 'age', 'married']}, True, 100, 0, sys.maxsize) assert result is not None and result.next().vertex_data is not None def test_get_tag_schema(): result = meta_client.get_tag_schema(space_name, 'player') - expect = {'name':6, 'age':2} + expect = {'name': SupportedType.STRING, 'age': SupportedType.INT, 'married': SupportedType.BOOL} assert result == expect def test_get_edge_schema(): result = meta_client.get_edge_schema(space_name, 'follow') - expect = {'degree': 5} + expect = {'degree': SupportedType.DOUBLE, 'time': SupportedType.TIMESTAMP} assert result == expect def test_get_edge_return_cols(): edge_item = meta_client.get_edge_item_from_cache(space_name, 'follow') edge_type = edge_item.edge_type entry_id = EntryId(edge_type=edge_type) - result = storage_client.get_edge_return_cols(space_name, {'follow':['degree']}) - expect = {edge_type:[PropDef(PropOwner.EDGE, entry_id, 'degree')]} + result = storage_client.get_edge_return_cols(space_name, {'follow':['degree', 'time']}) + expect = {edge_type:[PropDef(PropOwner.EDGE, entry_id, 'degree'), PropDef(PropOwner.EDGE, entry_id, 'time')]} assert result == expect def test_get_vertex_return_cols(): tag_item = meta_client.get_tag_item_from_cache(space_name, 'player') tag_id = tag_item.tag_id entry_id = EntryId(tag_id=tag_id) - result = storage_client.get_vertex_return_cols(space_name, {'player':['name', 'age']}) - expect = {tag_id:[PropDef(PropOwner.SOURCE, entry_id, 'name'), PropDef(PropOwner.SOURCE, entry_id, 'age')]} + result = storage_client.get_vertex_return_cols(space_name, {'player':['name', 'age', 'married']}) + expect = {tag_id:[PropDef(PropOwner.SOURCE, entry_id, 'name'), PropDef(PropOwner.SOURCE, entry_id, 'age'), PropDef(PropOwner.SOURCE, entry_id, 'married')]} assert result == expect def test_handle_result_codes(): From 66779edff90ff4979e073285d74cd10000fbba79 Mon Sep 17 00:00:00 2001 From: jievince Date: Sun, 30 Aug 2020 22:08:54 +0800 Subject: [PATCH 13/17] update tests --- examples/StorageClientExample.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index 5aa319d5..6314f0e8 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -10,6 +10,7 @@ """ import sys, getopt +import networkx as nx from meta.ttypes import ErrorCode sys.path.insert(0, '../') @@ -62,6 +63,8 @@ def process_edge(space, scan_edge_response): prop_value = prop.get_value() props[prop_name] = prop_value print(props) + # add edge and its properties to Graph G in NetworkX + G.add_edges_from([(srcId, dstId, props)]) def process_vertex(space, scan_vertex_response): result = scan_vertex_processor.process(space, scan_vertex_response) @@ -76,6 +79,8 @@ def process_vertex(space, scan_vertex_response): prop_value = prop.get_value() props[prop_name] = prop_value print(props) + # add node and its properties to Graph G in NetworkX + G.add_nodes_from([(vid, props)]) def get_return_cols(space): tag_items = meta_client.get_tags(space) @@ -116,6 +121,8 @@ def get_return_cols(space): vertex_return_cols, edge_return_cols = get_return_cols(space_to_read) all_cols = True + # initialize a Graph in NetworkX + G = nx.Graph() if space_to_read not in meta_client.get_parts_alloc_from_cache().keys(): raise Exception('spaceToRead %s is not found in nebula' % space_to_read) else: From 43edbab25f9e119ff3c3808e2e7e56c4ffb93fc0 Mon Sep 17 00:00:00 2001 From: jievince Date: Sun, 30 Aug 2020 22:09:36 +0800 Subject: [PATCH 14/17] update tests --- nebula/ngData/data.py | 3 +- nebula/ngMeta/MetaClient.py | 8 +- tests/test_StorageClient.py | 157 ++++++++++++++++++++++++++++++------ 3 files changed, 139 insertions(+), 29 deletions(-) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 30e9a4f2..31a70217 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -8,6 +8,7 @@ from common.ttypes import SupportedType import struct import six +from datetime import datetime class PropertyDef: def __init__(self, property_type, name): @@ -146,7 +147,7 @@ def get_int_property(self, name, value): def get_timestamp_property(self, name, value): val = self.read_compressed_int(value) - # val = datetime.fromtimestamp(val) + val = datetime.utcfromtimestamp(val).__str__() return Property(SupportedType.TIMESTAMP, name, val) def get_float_property(self, name, value): diff --git a/nebula/ngMeta/MetaClient.py b/nebula/ngMeta/MetaClient.py index ef679ca7..3d24aa09 100644 --- a/nebula/ngMeta/MetaClient.py +++ b/nebula/ngMeta/MetaClient.py @@ -291,7 +291,7 @@ def get_tags(self, space_name): print('get tags error, error code: ', list_tags_resp.code) return None - def get_tag(self, space_name, tag_name, version=-1): + def get_tag(self, space_name, tag_name, version=0): """ get tag schema of the given version Arguments: - space_name: name of the space @@ -312,7 +312,7 @@ def get_tag(self, space_name, tag_name, version=-1): else: return None - def get_tag_schema(self, space_name, tag_name, version=-1): + def get_tag_schema(self, space_name, tag_name, version=0): """ get tag schema columns of the given version Arguments: - space_name: name of the space @@ -394,7 +394,7 @@ def get_edges(self, space_name): print('get tags error, error code: ', list_edges_resp.code) return None - def get_edge(self, space_name, edge_name, version=-1): + def get_edge(self, space_name, edge_name, version=0): """ get edge schema of the given version Arguments: - space_name: name of the space @@ -417,7 +417,7 @@ def get_edge(self, space_name, edge_name, version=-1): print('get edge error, error code: ', get_edge_resp.code) return None - def get_edge_schema(self, space_name, edge_name, version=-1): + def get_edge_schema(self, space_name, edge_name, version=0): """ get edge schema columns of the given version Arguments: - space_name: name of the space diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index d1780825..5af5f50f 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -29,9 +29,10 @@ from nebula.Client import GraphClient from nebula.Common import * from nebula.ngStorage.StorageClient import StorageClient +from nebula.ngStorage.ngProcessor.ScanEdgeProcessor import ScanEdgeProcessor +from nebula.ngStorage.ngProcessor.ScanVertexProcessor import ScanVertexProcessor from nebula.ngMeta.MetaClient import MetaClient - def test_prepare(): try: client = GraphClient(ConnectionPool(host, graph_port)) @@ -51,60 +52,166 @@ def test_prepare(): time.sleep(5) resp = client.execute('CREATE TAG player(name string, age int, married bool)') assert resp.error_code == 0, resp.error_msg - resp = client.execute('CREATE EDGE follow(degree double, time timestamp)') + resp = client.execute('CREATE TAG team(name string, money double)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE EDGE follow(degree double, likeness int)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('CREATE EDGE serve(start timestamp, end timestamp)') assert resp.error_code == 0, resp.error_msg time.sleep(12) - resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 1:(\'Bob\', 18, FALSE)') + resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 101:(\'Bob\', 18, FALSE)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 102:(\'Tom\', 22, TRUE)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 103:(\'Jerry\', 19, FALSE)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX team(name, money) VALUES 201:(\'Red Bull\', 185.85)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT VERTEX team(name, money) VALUES 202:(\'River\', 567.93)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT EDGE follow(degree, likeness) VALUES 101->102:(94.7, 45)') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT EDGE follow(degree, likeness) VALUES 102->103:(86.3, 79)') assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT VERTEX player(name, age, married) VALUES 2:(\'Tome\', 22, TRUE)') + resp = client.execute('INSERT EDGE serve(start, end) VALUES 101->201:(\'2001-09-01 08:00:00\', \'2010-09-01 08:00:00\')') assert resp.error_code == 0, resp.error_msg - resp = client.execute('INSERT EDGE follow(degree, time) VALUES 1->2:(94.7, \'2010-09-01 08:00:00\')') + resp = client.execute('INSERT EDGE serve(start, end) VALUES 102->202:(\'1998-08-22 06:45:54\', \'2020-01-23 17:23:35\')') + assert resp.error_code == 0, resp.error_msg + resp = client.execute('INSERT EDGE serve(start, end) VALUES 103->201:(\'2006-11-18 13:28:29\', \'2009-12-12 12:21:46\')') assert resp.error_code == 0, resp.error_msg except Exception as ex: print(ex) client.sign_out() assert False +def check_result(result, expect): + if len(result) != len(expect): + print('len(result) != len(expect)') + return False + for name in expect.keys(): + if name not in result.keys(): + print(name, ' not in result') + return False + if len(result[name]) != len(expect[name]): + print('len(result[%s]) != len(expect[%s])' % (name, name)) + return False + for value in expect[name]: + if value not in result[name]: + print(value, ' not in result[%s]' % name) + return False + return True + +def get_result(space, scan_response_iter, is_edge): + scan_response = scan_response_iter.next() + if scan_response is None: + assert False + result = {} + result.update(process_response(space, scan_response, is_edge)) + while scan_response_iter.has_next(): + scan_response = scan_response_iter.next() + if scan_response is None: + assert False + result.update(process_response(space, scan_response, is_edge)) + return result + +def process_response(space, scan_response, is_edge): + if is_edge: + result = scan_edge_processor.process(space, scan_response) + else: + result = scan_vertex_processor.process(space, scan_response) + datas = {} + for name, rows in result._rows.items(): + data = [] + for row in rows: + props = {} + for prop in row._default_properties: + props[prop.get_name()] = prop.get_value() + for prop in row._properties: + props[prop.get_name()] = prop.get_value() + data.append(props) + datas[name] = data + return datas + def test_scan_edge(): - result = storage_client.scan_edge(space_name, {'follow':['degree', 'time']}, True, 100, 0, sys.maxsize) - assert result is not None and result.next().edge_data is not None + scan_edge_response_iter = storage_client.scan_edge(space_name, {'follow': ['degree', 'likeness'], 'serve': ['start', 'end']}, True, 100, 0, sys.maxsize) + result = get_result(space_name, scan_edge_response_iter, True) + expect = {'follow': [{'_src': 101, '_edge': 'follow', '_dst': 102, 'degree': 94.7, 'likeness': 45}, + {'_src': 102, '_edge': 'follow', '_dst': 103, 'degree': 86.3, 'likeness': 79}], + 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': '2001-09-01 08:00:00', 'end': '2010-09-01 08:00:00'}, + {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': '1998-08-22 06:45:54', 'end': '2020-01-23 17:23:35'}, + {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': '2006-11-18 13:28:29', 'end': '2009-12-12 12:21:46'}]} + assert check_result(result, expect) def test_scan_vertex(): - result = storage_client.scan_vertex(space_name, {'player':['name', 'age', 'married']}, True, 100, 0, sys.maxsize) - assert result is not None and result.next().vertex_data is not None + scan_vertex_response_iter = storage_client.scan_vertex(space_name, {'player': ['name', 'age', 'married'], 'team': ['name', 'money']}, True, 100, 0, sys.maxsize) + result = get_result(space_name, scan_vertex_response_iter, False) + expect = {'player': [{'_vid': 101, '_tag': 'player', 'name': 'Bob', 'age': 18, 'married': False}, + {'_vid': 102, '_tag': 'player', 'name': 'Tom', 'age': 22, 'married': True}, + {'_vid': 103, '_tag': 'player', 'name': 'Jerry', 'age': 19, 'married': False}], + 'team': [{'_vid': 201, '_tag': 'team', 'name': 'Red Bull', 'money': 185.85}, + {'_vid': 202, '_tag': 'team', 'name': 'River', 'money': 567.93}]} + assert check_result(result, expect) def test_scan_part_edge(): - result = storage_client.scan_part_edge(space_name, 1, {'follow':['degree', 'time']}, True, 100, 0, sys.maxsize) - assert result is not None and result.next().edge_data is not None + scan_edge_response_iter = storage_client.scan_part_edge(space_name, 1, {'follow': ['degree', 'likeness'], 'serve': ['start', 'end']}, True, 100, 0, sys.maxsize) + result = get_result(space_name, scan_edge_response_iter, True) + expect = {'follow': [{'_src': 101, '_edge': 'follow', '_dst': 102, 'degree': 94.7, 'likeness': 45}, + {'_src': 102, '_edge': 'follow', '_dst': 103, 'degree': 86.3, 'likeness': 79}], + 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': '2001-09-01 08:00:00', 'end': '2010-09-01 08:00:00'}, + {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': '1998-08-22 06:45:54', 'end': '2020-01-23 17:23:35'}, + {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': '2006-11-18 13:28:29', 'end': '2009-12-12 12:21:46'}]} + assert check_result(result, expect) def test_scan_part_vertex(): - result = storage_client.scan_part_vertex(space_name, 1, {'player':['name', 'age', 'married']}, True, 100, 0, sys.maxsize) - assert result is not None and result.next().vertex_data is not None + scan_vertex_response_iter = storage_client.scan_part_vertex(space_name, 1, {'player':['name', 'age', 'married'], 'team': ['name', 'money']}, True, 100, 0, sys.maxsize) + result = get_result(space_name, scan_vertex_response_iter, False) + expect = {'player': [{'_vid': 101, '_tag': 'player', 'name': 'Bob', 'age': 18, 'married': False}, + {'_vid': 102, '_tag': 'player', 'name': 'Tom', 'age': 22, 'married': True}, + {'_vid': 103, '_tag': 'player', 'name': 'Jerry', 'age': 19, 'married': False}], + 'team': [{'_vid': 201, '_tag': 'team', 'name': 'Red Bull', 'money': 185.85}, + {'_vid': 202, '_tag': 'team', 'name': 'River', 'money': 567.93}]} + assert check_result(result, expect) def test_get_tag_schema(): result = meta_client.get_tag_schema(space_name, 'player') expect = {'name': SupportedType.STRING, 'age': SupportedType.INT, 'married': SupportedType.BOOL} assert result == expect + result = meta_client.get_tag_schema(space_name, 'team') + expect = {'name': SupportedType.STRING, 'money': SupportedType.DOUBLE} + assert result == expect + def test_get_edge_schema(): result = meta_client.get_edge_schema(space_name, 'follow') - expect = {'degree': SupportedType.DOUBLE, 'time': SupportedType.TIMESTAMP} + expect = {'degree': SupportedType.DOUBLE, 'likeness': SupportedType.INT} + assert result == expect + + result = meta_client.get_edge_schema(space_name, 'serve') + expect = {'start': SupportedType.TIMESTAMP, 'end': SupportedType.TIMESTAMP} assert result == expect def test_get_edge_return_cols(): - edge_item = meta_client.get_edge_item_from_cache(space_name, 'follow') - edge_type = edge_item.edge_type - entry_id = EntryId(edge_type=edge_type) - result = storage_client.get_edge_return_cols(space_name, {'follow':['degree', 'time']}) - expect = {edge_type:[PropDef(PropOwner.EDGE, entry_id, 'degree'), PropDef(PropOwner.EDGE, entry_id, 'time')]} + edge_item_1 = meta_client.get_edge_item_from_cache(space_name, 'follow') + edge_type_1 = edge_item_1.edge_type + entry_id_1 = EntryId(edge_type=edge_type_1) + edge_item_2 = meta_client.get_edge_item_from_cache(space_name, 'serve') + edge_type_2 = edge_item_2.edge_type + entry_id_2 = EntryId(edge_type=edge_type_2) + result = storage_client.get_edge_return_cols(space_name, {'follow': ['degree', 'likeness'], 'serve': ['start', 'end']}) + expect = {edge_type_1:[PropDef(PropOwner.EDGE, entry_id_1, 'degree'), PropDef(PropOwner.EDGE, entry_id_1, 'likeness')], + edge_type_2:[PropDef(PropOwner.EDGE, entry_id_2, 'start'), PropDef(PropOwner.EDGE, entry_id_2, 'end')]} assert result == expect def test_get_vertex_return_cols(): - tag_item = meta_client.get_tag_item_from_cache(space_name, 'player') - tag_id = tag_item.tag_id - entry_id = EntryId(tag_id=tag_id) - result = storage_client.get_vertex_return_cols(space_name, {'player':['name', 'age', 'married']}) - expect = {tag_id:[PropDef(PropOwner.SOURCE, entry_id, 'name'), PropDef(PropOwner.SOURCE, entry_id, 'age'), PropDef(PropOwner.SOURCE, entry_id, 'married')]} + tag_item_1 = meta_client.get_tag_item_from_cache(space_name, 'player') + tag_id_1 = tag_item_1.tag_id + entry_id_1 = EntryId(tag_id=tag_id_1) + tag_item_2 = meta_client.get_tag_item_from_cache(space_name, 'team') + tag_id_2 = tag_item_2.tag_id + entry_id_2 = EntryId(tag_id=tag_id_2) + result = storage_client.get_vertex_return_cols(space_name, {'player': ['name', 'age', 'married'], 'team': ['name', 'money']}) + expect = {tag_id_1:[PropDef(PropOwner.SOURCE, entry_id_1, 'name'), PropDef(PropOwner.SOURCE, entry_id_1, 'age'), PropDef(PropOwner.SOURCE, entry_id_1, 'married')], + tag_id_2:[PropDef(PropOwner.SOURCE, entry_id_2, 'name'), PropDef(PropOwner.SOURCE, entry_id_2, 'money')]} assert result == expect def test_handle_result_codes(): @@ -122,3 +229,5 @@ def test_handle_result_codes(): meta_client = MetaClient([(host, meta_port)]) meta_client.connect() storage_client = StorageClient(meta_client) +scan_edge_processor = ScanEdgeProcessor(meta_client) +scan_vertex_processor = ScanVertexProcessor(meta_client) From 68ebb7c6e2d5bdee8b0e64168b16e941bcef7d7e Mon Sep 17 00:00:00 2001 From: jievince Date: Sun, 30 Aug 2020 22:14:13 +0800 Subject: [PATCH 15/17] update examples --- examples/StorageClientExample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index 6314f0e8..ee6c3854 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -104,7 +104,7 @@ def get_return_cols(space): if __name__ == '__main__': - # initialize a MetaClient to establish connections with meta servers + # initialize a MetaClient to establish a connection with the meta server meta_client = MetaClient([(sys.argv[1], sys.argv[2])]) code = meta_client.connect() if code == ErrorCode.E_FAIL_TO_CONNECT: From fd0a54314fe9d7802b3885f371751e5f66db84d5 Mon Sep 17 00:00:00 2001 From: jievince Date: Sun, 30 Aug 2020 23:07:18 +0800 Subject: [PATCH 16/17] improve has_next() --- examples/StorageClientExample.py | 10 ---------- nebula/ngStorage/StorageClient.py | 6 +++--- tests/test_StorageClient.py | 4 ---- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/examples/StorageClientExample.py b/examples/StorageClientExample.py index ee6c3854..d7e1e341 100644 --- a/examples/StorageClientExample.py +++ b/examples/StorageClientExample.py @@ -23,11 +23,6 @@ def scan_edge(space, return_cols, all_cols): scan_edge_response_iter = storage_client.scan_edge(space, return_cols, all_cols, 100, 0, sys.maxsize) - scan_edge_response = scan_edge_response_iter.next() - if scan_edge_response is None: - print('scan_edge_response is None') - return - process_edge(space, scan_edge_response) while scan_edge_response_iter.has_next(): scan_edge_response = scan_edge_response_iter.next() if scan_edge_response is None: @@ -37,11 +32,6 @@ def scan_edge(space, return_cols, all_cols): def scan_vertex(space, return_cols, all_cols): scan_vertex_response_iter = storage_client.scan_vertex(space, return_cols, all_cols, 100, 0, sys.maxsize) - scan_vertex_response = scan_vertex_response_iter.next() - if scan_vertex_response is None: - print('scan_vertex_vesponse is None') - return - process_vertex(space, scan_vertex_response) while scan_vertex_response_iter.has_next(): scan_vertex_response = scan_vertex_response_iter.next() if scan_vertex_response is None: diff --git a/nebula/ngStorage/StorageClient.py b/nebula/ngStorage/StorageClient.py index efad5a44..1552f6ef 100644 --- a/nebula/ngStorage/StorageClient.py +++ b/nebula/ngStorage/StorageClient.py @@ -71,7 +71,7 @@ def next(self): if not self._client_dad.is_successfully(scan_edge_response): self._leader, self._client = self._client_dad.handle_result_codes(scan_edge_response.result.failed_codes, self._space) - self._haveNext = False + self._have_next = False return None else: return scan_edge_response @@ -90,7 +90,7 @@ def __init__(self, client_dad, space, leader, scan_vertex_request, client): self._have_next = True def has_next(self): - return self._haveNext + return self._have_next def next(self): self._scan_vertex_request.cursor = self._cursor @@ -98,7 +98,7 @@ def next(self): if scan_vertex_response is None: raise Exception('scan_vertex_reponse is None') self._cursor = scan_vertex_response.next_cursor - self._haveNext = scan_vertex_response.has_next + self._have_next = scan_vertex_response.has_next if not self._client_dad.is_successfully(scan_vertex_response): print('scan_vertex_response is not successfully, failed_codes: ', scan_vertex_response.result.failed_codes) diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 5af5f50f..8dc29103 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -102,11 +102,7 @@ def check_result(result, expect): return True def get_result(space, scan_response_iter, is_edge): - scan_response = scan_response_iter.next() - if scan_response is None: - assert False result = {} - result.update(process_response(space, scan_response, is_edge)) while scan_response_iter.has_next(): scan_response = scan_response_iter.next() if scan_response is None: From 3713f7cd9f79399fbb0d92b63f83cf20508f247e Mon Sep 17 00:00:00 2001 From: jievince Date: Mon, 31 Aug 2020 11:04:49 +0800 Subject: [PATCH 17/17] fix timestamp --- nebula/ngData/data.py | 1 - tests/test_StorageClient.py | 17 +++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/nebula/ngData/data.py b/nebula/ngData/data.py index 31a70217..1248e0a1 100644 --- a/nebula/ngData/data.py +++ b/nebula/ngData/data.py @@ -147,7 +147,6 @@ def get_int_property(self, name, value): def get_timestamp_property(self, name, value): val = self.read_compressed_int(value) - val = datetime.utcfromtimestamp(val).__str__() return Property(SupportedType.TIMESTAMP, name, val) def get_float_property(self, name, value): diff --git a/tests/test_StorageClient.py b/tests/test_StorageClient.py index 8dc29103..a3cf6403 100644 --- a/tests/test_StorageClient.py +++ b/tests/test_StorageClient.py @@ -15,6 +15,7 @@ import os import time import threading +import datetime sys.path.insert(0, '../') from storage.ttypes import EntryId @@ -128,14 +129,18 @@ def process_response(space, scan_response, is_edge): datas[name] = data return datas +def ts(date_str): + dt = datetime.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") + return dt.replace(tzinfo=datetime.timezone.utc).timestamp() + def test_scan_edge(): scan_edge_response_iter = storage_client.scan_edge(space_name, {'follow': ['degree', 'likeness'], 'serve': ['start', 'end']}, True, 100, 0, sys.maxsize) result = get_result(space_name, scan_edge_response_iter, True) expect = {'follow': [{'_src': 101, '_edge': 'follow', '_dst': 102, 'degree': 94.7, 'likeness': 45}, {'_src': 102, '_edge': 'follow', '_dst': 103, 'degree': 86.3, 'likeness': 79}], - 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': '2001-09-01 08:00:00', 'end': '2010-09-01 08:00:00'}, - {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': '1998-08-22 06:45:54', 'end': '2020-01-23 17:23:35'}, - {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': '2006-11-18 13:28:29', 'end': '2009-12-12 12:21:46'}]} + 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': ts('2001-09-01 08:00:00'), 'end': ts('2010-09-01 08:00:00')}, + {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': ts('1998-08-22 06:45:54'), 'end': ts('2020-01-23 17:23:35')}, + {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': ts('2006-11-18 13:28:29'), 'end': ts('2009-12-12 12:21:46')}]} assert check_result(result, expect) def test_scan_vertex(): @@ -153,9 +158,9 @@ def test_scan_part_edge(): result = get_result(space_name, scan_edge_response_iter, True) expect = {'follow': [{'_src': 101, '_edge': 'follow', '_dst': 102, 'degree': 94.7, 'likeness': 45}, {'_src': 102, '_edge': 'follow', '_dst': 103, 'degree': 86.3, 'likeness': 79}], - 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': '2001-09-01 08:00:00', 'end': '2010-09-01 08:00:00'}, - {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': '1998-08-22 06:45:54', 'end': '2020-01-23 17:23:35'}, - {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': '2006-11-18 13:28:29', 'end': '2009-12-12 12:21:46'}]} + 'serve': [{'_src': 101, '_edge': 'serve', '_dst': 201, 'start': ts('2001-09-01 08:00:00'), 'end': ts('2010-09-01 08:00:00')}, + {'_src': 102, '_edge': 'serve', '_dst': 202, 'start': ts('1998-08-22 06:45:54'), 'end': ts('2020-01-23 17:23:35')}, + {'_src': 103, '_edge': 'serve', '_dst': 201, 'start': ts('2006-11-18 13:28:29'), 'end': ts('2009-12-12 12:21:46')}]} assert check_result(result, expect) def test_scan_part_vertex():