From 7672e5957e59bb81cd4e555fc9ea6fbd2f41d402 Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:14:53 -0700 Subject: [PATCH 1/7] Update thrift definitions Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- .../thrift_api/TCLIService/TCLIService-remote | 9 +- .../sql/thrift_api/TCLIService/TCLIService.py | 191 +- .../sql/thrift_api/TCLIService/constants.py | 2 +- .../sql/thrift_api/TCLIService/ttypes.py | 4124 ++++++++++++++--- 4 files changed, 3378 insertions(+), 948 deletions(-) diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote index 5271f9556..f6f1eba7b 100755 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.18.1) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -45,7 +45,6 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)') print(' TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)') print(' TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)') - print(' TDBSqlGetLoadInformationResp GetLoadInformation(TDBSqlGetLoadInformationReq req)') print('') sys.exit(0) @@ -251,12 +250,6 @@ elif cmd == 'RenewDelegationToken': sys.exit(1) pp.pprint(client.RenewDelegationToken(eval(args[0]),)) -elif cmd == 'GetLoadInformation': - if len(args) != 1: - print('GetLoadInformation requires 1 args') - sys.exit(1) - pp.pprint(client.GetLoadInformation(eval(args[0]),)) - else: print('Unrecognized method %s' % cmd) sys.exit(1) diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py index 6dde67026..08f0913b1 100644 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.18.1) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -187,14 +187,6 @@ def RenewDelegationToken(self, req): """ pass - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - pass - class Client(Iface): def __init__(self, iprot, oprot=None): @@ -875,38 +867,6 @@ def recv_RenewDelegationToken(self): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "RenewDelegationToken failed: unknown result") - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - self.send_GetLoadInformation(req) - return self.recv_GetLoadInformation() - - def send_GetLoadInformation(self, req): - self._oprot.writeMessageBegin('GetLoadInformation', TMessageType.CALL, self._seqid) - args = GetLoadInformation_args() - args.req = req - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_GetLoadInformation(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = GetLoadInformation_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "GetLoadInformation failed: unknown result") - class Processor(Iface, TProcessor): def __init__(self, handler): @@ -933,7 +893,6 @@ def __init__(self, handler): self._processMap["GetDelegationToken"] = Processor.process_GetDelegationToken self._processMap["CancelDelegationToken"] = Processor.process_CancelDelegationToken self._processMap["RenewDelegationToken"] = Processor.process_RenewDelegationToken - self._processMap["GetLoadInformation"] = Processor.process_GetLoadInformation self._on_message_begin = None def on_message_begin(self, func): @@ -1439,29 +1398,6 @@ def process_RenewDelegationToken(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() - def process_GetLoadInformation(self, seqid, iprot, oprot): - args = GetLoadInformation_args() - args.read(iprot) - iprot.readMessageEnd() - result = GetLoadInformation_result() - try: - result.success = self._handler.GetLoadInformation(args.req) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("GetLoadInformation", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - # HELPER FUNCTIONS AND STRUCTURES @@ -4088,130 +4024,5 @@ def __ne__(self, other): RenewDelegationToken_result.thrift_spec = ( (0, TType.STRUCT, 'success', [TRenewDelegationTokenResp, None], None, ), # 0 ) - - -class GetLoadInformation_args(object): - """ - Attributes: - - req - - """ - - - def __init__(self, req=None,): - self.req = req - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.req = TDBSqlGetLoadInformationReq() - self.req.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_args') - if self.req is not None: - oprot.writeFieldBegin('req', TType.STRUCT, 1) - self.req.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_args) -GetLoadInformation_args.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'req', [TDBSqlGetLoadInformationReq, None], None, ), # 1 -) - - -class GetLoadInformation_result(object): - """ - Attributes: - - success - - """ - - - def __init__(self, success=None,): - self.success = success - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = TDBSqlGetLoadInformationResp() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_result) -GetLoadInformation_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [TDBSqlGetLoadInformationResp, None], None, ), # 0 -) fix_spec(all_structs) del all_structs diff --git a/src/databricks/sql/thrift_api/TCLIService/constants.py b/src/databricks/sql/thrift_api/TCLIService/constants.py index 66dfc322b..426e5ae16 100644 --- a/src/databricks/sql/thrift_api/TCLIService/constants.py +++ b/src/databricks/sql/thrift_api/TCLIService/constants.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.18.1) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # diff --git a/src/databricks/sql/thrift_api/TCLIService/ttypes.py b/src/databricks/sql/thrift_api/TCLIService/ttypes.py index 07ecfe48d..75fa1f204 100644 --- a/src/databricks/sql/thrift_api/TCLIService/ttypes.py +++ b/src/databricks/sql/thrift_api/TCLIService/ttypes.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.18.1) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -178,6 +178,39 @@ class TSparkRowSetType(object): } +class TDBSqlCompressionCodec(object): + NONE = 0 + LZ4_FRAME = 1 + LZ4_BLOCK = 2 + + _VALUES_TO_NAMES = { + 0: "NONE", + 1: "LZ4_FRAME", + 2: "LZ4_BLOCK", + } + + _NAMES_TO_VALUES = { + "NONE": 0, + "LZ4_FRAME": 1, + "LZ4_BLOCK": 2, + } + + +class TDBSqlArrowLayout(object): + ARROW_BATCH = 0 + ARROW_STREAMING = 1 + + _VALUES_TO_NAMES = { + 0: "ARROW_BATCH", + 1: "ARROW_STREAMING", + } + + _NAMES_TO_VALUES = { + "ARROW_BATCH": 0, + "ARROW_STREAMING": 1, + } + + class TOperationIdempotencyType(object): UNKNOWN = 0 NON_IDEMPOTENT = 1 @@ -529,6 +562,18 @@ class TCloudFetchDisabledReason(object): } +class TDBSqlManifestFileFormat(object): + THRIFT_GET_RESULT_SET_METADATA_RESP = 0 + + _VALUES_TO_NAMES = { + 0: "THRIFT_GET_RESULT_SET_METADATA_RESP", + } + + _NAMES_TO_VALUES = { + "THRIFT_GET_RESULT_SET_METADATA_RESP": 0, + } + + class TFetchOrientation(object): FETCH_NEXT = 0 FETCH_PRIOR = 1 @@ -556,6 +601,27 @@ class TFetchOrientation(object): } +class TDBSqlFetchDisposition(object): + DISPOSITION_UNSPECIFIED = 0 + DISPOSITION_INLINE = 1 + DISPOSITION_EXTERNAL_LINKS = 2 + DISPOSITION_INTERNAL_DBFS = 3 + + _VALUES_TO_NAMES = { + 0: "DISPOSITION_UNSPECIFIED", + 1: "DISPOSITION_INLINE", + 2: "DISPOSITION_EXTERNAL_LINKS", + 3: "DISPOSITION_INTERNAL_DBFS", + } + + _NAMES_TO_VALUES = { + "DISPOSITION_UNSPECIFIED": 0, + "DISPOSITION_INLINE": 1, + "DISPOSITION_EXTERNAL_LINKS": 2, + "DISPOSITION_INTERNAL_DBFS": 3, + } + + class TJobExecutionStatus(object): IN_PROGRESS = 0 COMPLETE = 1 @@ -2841,6 +2907,270 @@ def __ne__(self, other): return not (self == other) +class TDBSqlJsonArrayFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlJsonArrayFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlCsvFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlCsvFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlArrowFormat(object): + """ + Attributes: + - arrowLayout + - compressionCodec + + """ + + + def __init__(self, arrowLayout=None, compressionCodec=None,): + self.arrowLayout = arrowLayout + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.arrowLayout = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlArrowFormat') + if self.arrowLayout is not None: + oprot.writeFieldBegin('arrowLayout', TType.I32, 1) + oprot.writeI32(self.arrowLayout) + oprot.writeFieldEnd() + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 2) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlResultFormat(object): + """ + Attributes: + - arrowFormat + - csvFormat + - jsonArrayFormat + + """ + + + def __init__(self, arrowFormat=None, csvFormat=None, jsonArrayFormat=None,): + self.arrowFormat = arrowFormat + self.csvFormat = csvFormat + self.jsonArrayFormat = jsonArrayFormat + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.arrowFormat = TDBSqlArrowFormat() + self.arrowFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.csvFormat = TDBSqlCsvFormat() + self.csvFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.jsonArrayFormat = TDBSqlJsonArrayFormat() + self.jsonArrayFormat.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlResultFormat') + if self.arrowFormat is not None: + oprot.writeFieldBegin('arrowFormat', TType.STRUCT, 1) + self.arrowFormat.write(oprot) + oprot.writeFieldEnd() + if self.csvFormat is not None: + oprot.writeFieldBegin('csvFormat', TType.STRUCT, 2) + self.csvFormat.write(oprot) + oprot.writeFieldEnd() + if self.jsonArrayFormat is not None: + oprot.writeFieldBegin('jsonArrayFormat', TType.STRUCT, 3) + self.jsonArrayFormat.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TSparkArrowBatch(object): """ Attributes: @@ -2921,16 +3251,18 @@ class TSparkArrowResultLink(object): - startRowOffset - rowCount - bytesNum + - httpHeaders """ - def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None,): + def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None, httpHeaders=None,): self.fileLink = fileLink self.expiryTime = expiryTime self.startRowOffset = startRowOffset self.rowCount = rowCount self.bytesNum = bytesNum + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -2966,6 +3298,17 @@ def read(self, iprot): self.bytesNum = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype105, _vtype106, _size104) = iprot.readMapBegin() + for _i108 in range(_size104): + _key109 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val110 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key109] = _val110 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -2996,6 +3339,14 @@ def write(self, oprot): oprot.writeFieldBegin('bytesNum', TType.I64, 5) oprot.writeI64(self.bytesNum) oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 6) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter111, viter112 in self.httpHeaders.items(): + oprot.writeString(kiter111.encode('utf-8') if sys.version_info[0] == 2 else kiter111) + oprot.writeString(viter112.encode('utf-8') if sys.version_info[0] == 2 else viter112) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -3032,16 +3383,22 @@ class TDBSqlCloudResultFile(object): - rowCount - uncompressedBytes - compressedBytes + - fileLink + - linkExpiryTime + - httpHeaders """ - def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None,): + def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None, fileLink=None, linkExpiryTime=None, httpHeaders=None,): self.filePath = filePath self.startRowOffset = startRowOffset self.rowCount = rowCount self.uncompressedBytes = uncompressedBytes self.compressedBytes = compressedBytes + self.fileLink = fileLink + self.linkExpiryTime = linkExpiryTime + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3077,6 +3434,27 @@ def read(self, iprot): self.compressedBytes = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.fileLink = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.linkExpiryTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype114, _vtype115, _size113) = iprot.readMapBegin() + for _i117 in range(_size113): + _key118 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val119 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key118] = _val119 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -3107,20 +3485,26 @@ def write(self, oprot): oprot.writeFieldBegin('compressedBytes', TType.I64, 5) oprot.writeI64(self.compressedBytes) oprot.writeFieldEnd() + if self.fileLink is not None: + oprot.writeFieldBegin('fileLink', TType.STRING, 6) + oprot.writeString(self.fileLink.encode('utf-8') if sys.version_info[0] == 2 else self.fileLink) + oprot.writeFieldEnd() + if self.linkExpiryTime is not None: + oprot.writeFieldBegin('linkExpiryTime', TType.I64, 7) + oprot.writeI64(self.linkExpiryTime) + oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 8) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter120, viter121 in self.httpHeaders.items(): + oprot.writeString(kiter120.encode('utf-8') if sys.version_info[0] == 2 else kiter120) + oprot.writeString(viter121.encode('utf-8') if sys.version_info[0] == 2 else viter121) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): - if self.filePath is None: - raise TProtocolException(message='Required field filePath is unset!') - if self.startRowOffset is None: - raise TProtocolException(message='Required field startRowOffset is unset!') - if self.rowCount is None: - raise TProtocolException(message='Required field rowCount is unset!') - if self.uncompressedBytes is None: - raise TProtocolException(message='Required field uncompressedBytes is unset!') - if self.compressedBytes is None: - raise TProtocolException(message='Required field compressedBytes is unset!') return def __repr__(self): @@ -3145,11 +3529,12 @@ class TRowSet(object): - columnCount - arrowBatches - resultLinks + - cloudFetchResults """ - def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None,): + def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None, cloudFetchResults=None,): self.startRowOffset = startRowOffset self.rows = rows self.columns = columns @@ -3157,6 +3542,7 @@ def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=N self.columnCount = columnCount self.arrowBatches = arrowBatches self.resultLinks = resultLinks + self.cloudFetchResults = cloudFetchResults def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3175,22 +3561,22 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype107, _size104) = iprot.readListBegin() - for _i108 in range(_size104): - _elem109 = TRow() - _elem109.read(iprot) - self.rows.append(_elem109) + (_etype125, _size122) = iprot.readListBegin() + for _i126 in range(_size122): + _elem127 = TRow() + _elem127.read(iprot) + self.rows.append(_elem127) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype113, _size110) = iprot.readListBegin() - for _i114 in range(_size110): - _elem115 = TColumn() - _elem115.read(iprot) - self.columns.append(_elem115) + (_etype131, _size128) = iprot.readListBegin() + for _i132 in range(_size128): + _elem133 = TColumn() + _elem133.read(iprot) + self.columns.append(_elem133) iprot.readListEnd() else: iprot.skip(ftype) @@ -3207,22 +3593,33 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.arrowBatches = [] - (_etype119, _size116) = iprot.readListBegin() - for _i120 in range(_size116): - _elem121 = TSparkArrowBatch() - _elem121.read(iprot) - self.arrowBatches.append(_elem121) + (_etype137, _size134) = iprot.readListBegin() + for _i138 in range(_size134): + _elem139 = TSparkArrowBatch() + _elem139.read(iprot) + self.arrowBatches.append(_elem139) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 1282: if ftype == TType.LIST: self.resultLinks = [] - (_etype125, _size122) = iprot.readListBegin() - for _i126 in range(_size122): - _elem127 = TSparkArrowResultLink() - _elem127.read(iprot) - self.resultLinks.append(_elem127) + (_etype143, _size140) = iprot.readListBegin() + for _i144 in range(_size140): + _elem145 = TSparkArrowResultLink() + _elem145.read(iprot) + self.resultLinks.append(_elem145) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3329: + if ftype == TType.LIST: + self.cloudFetchResults = [] + (_etype149, _size146) = iprot.readListBegin() + for _i150 in range(_size146): + _elem151 = TDBSqlCloudResultFile() + _elem151.read(iprot) + self.cloudFetchResults.append(_elem151) iprot.readListEnd() else: iprot.skip(ftype) @@ -3243,15 +3640,15 @@ def write(self, oprot): if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rows)) - for iter128 in self.rows: - iter128.write(oprot) + for iter152 in self.rows: + iter152.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.columns)) - for iter129 in self.columns: - iter129.write(oprot) + for iter153 in self.columns: + iter153.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.binaryColumns is not None: @@ -3265,15 +3662,22 @@ def write(self, oprot): if self.arrowBatches is not None: oprot.writeFieldBegin('arrowBatches', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.arrowBatches)) - for iter130 in self.arrowBatches: - iter130.write(oprot) + for iter154 in self.arrowBatches: + iter154.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.resultLinks is not None: oprot.writeFieldBegin('resultLinks', TType.LIST, 1282) oprot.writeListBegin(TType.STRUCT, len(self.resultLinks)) - for iter131 in self.resultLinks: - iter131.write(oprot) + for iter155 in self.resultLinks: + iter155.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.cloudFetchResults is not None: + oprot.writeFieldBegin('cloudFetchResults', TType.LIST, 3329) + oprot.writeListBegin(TType.STRUCT, len(self.cloudFetchResults)) + for iter156 in self.cloudFetchResults: + iter156.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3337,11 +3741,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.properties = {} - (_ktype133, _vtype134, _size132) = iprot.readMapBegin() - for _i136 in range(_size132): - _key137 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val138 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.properties[_key137] = _val138 + (_ktype158, _vtype159, _size157) = iprot.readMapBegin() + for _i161 in range(_size157): + _key162 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val163 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.properties[_key162] = _val163 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3371,9 +3775,9 @@ def write(self, oprot): if self.properties is not None: oprot.writeFieldBegin('properties', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) - for kiter139, viter140 in self.properties.items(): - oprot.writeString(kiter139.encode('utf-8') if sys.version_info[0] == 2 else kiter139) - oprot.writeString(viter140.encode('utf-8') if sys.version_info[0] == 2 else viter140) + for kiter164, viter165 in self.properties.items(): + oprot.writeString(kiter164.encode('utf-8') if sys.version_info[0] == 2 else kiter164) + oprot.writeString(viter165.encode('utf-8') if sys.version_info[0] == 2 else viter165) oprot.writeMapEnd() oprot.writeFieldEnd() if self.viewSchema is not None: @@ -3725,22 +4129,22 @@ def read(self, iprot): if fid == 1: if ftype == TType.MAP: self.confs = {} - (_ktype142, _vtype143, _size141) = iprot.readMapBegin() - for _i145 in range(_size141): - _key146 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val147 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confs[_key146] = _val147 + (_ktype167, _vtype168, _size166) = iprot.readMapBegin() + for _i170 in range(_size166): + _key171 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val172 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confs[_key171] = _val172 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.tempViews = [] - (_etype151, _size148) = iprot.readListBegin() - for _i152 in range(_size148): - _elem153 = TDBSqlTempView() - _elem153.read(iprot) - self.tempViews.append(_elem153) + (_etype176, _size173) = iprot.readListBegin() + for _i177 in range(_size173): + _elem178 = TDBSqlTempView() + _elem178.read(iprot) + self.tempViews.append(_elem178) iprot.readListEnd() else: iprot.skip(ftype) @@ -3763,23 +4167,23 @@ def read(self, iprot): elif fid == 6: if ftype == TType.LIST: self.expressionsInfos = [] - (_etype157, _size154) = iprot.readListBegin() - for _i158 in range(_size154): - _elem159 = TExpressionInfo() - _elem159.read(iprot) - self.expressionsInfos.append(_elem159) + (_etype182, _size179) = iprot.readListBegin() + for _i183 in range(_size179): + _elem184 = TExpressionInfo() + _elem184.read(iprot) + self.expressionsInfos.append(_elem184) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 7: if ftype == TType.MAP: self.internalConfs = {} - (_ktype161, _vtype162, _size160) = iprot.readMapBegin() - for _i164 in range(_size160): - _key165 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val166 = TDBSqlConfValue() - _val166.read(iprot) - self.internalConfs[_key165] = _val166 + (_ktype186, _vtype187, _size185) = iprot.readMapBegin() + for _i189 in range(_size185): + _key190 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val191 = TDBSqlConfValue() + _val191.read(iprot) + self.internalConfs[_key190] = _val191 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3796,16 +4200,16 @@ def write(self, oprot): if self.confs is not None: oprot.writeFieldBegin('confs', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confs)) - for kiter167, viter168 in self.confs.items(): - oprot.writeString(kiter167.encode('utf-8') if sys.version_info[0] == 2 else kiter167) - oprot.writeString(viter168.encode('utf-8') if sys.version_info[0] == 2 else viter168) + for kiter192, viter193 in self.confs.items(): + oprot.writeString(kiter192.encode('utf-8') if sys.version_info[0] == 2 else kiter192) + oprot.writeString(viter193.encode('utf-8') if sys.version_info[0] == 2 else viter193) oprot.writeMapEnd() oprot.writeFieldEnd() if self.tempViews is not None: oprot.writeFieldBegin('tempViews', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.tempViews)) - for iter169 in self.tempViews: - iter169.write(oprot) + for iter194 in self.tempViews: + iter194.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.currentDatabase is not None: @@ -3823,16 +4227,16 @@ def write(self, oprot): if self.expressionsInfos is not None: oprot.writeFieldBegin('expressionsInfos', TType.LIST, 6) oprot.writeListBegin(TType.STRUCT, len(self.expressionsInfos)) - for iter170 in self.expressionsInfos: - iter170.write(oprot) + for iter195 in self.expressionsInfos: + iter195.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.internalConfs is not None: oprot.writeFieldBegin('internalConfs', TType.MAP, 7) oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(self.internalConfs)) - for kiter171, viter172 in self.internalConfs.items(): - oprot.writeString(kiter171.encode('utf-8') if sys.version_info[0] == 2 else kiter171) - viter172.write(oprot) + for kiter196, viter197 in self.internalConfs.items(): + oprot.writeString(kiter196.encode('utf-8') if sys.version_info[0] == 2 else kiter196) + viter197.write(oprot) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3862,18 +4266,20 @@ class TStatus(object): - errorCode - errorMessage - displayMessage + - errorDetailsJson - responseValidation """ - def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, responseValidation=None,): + def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, errorDetailsJson=None, responseValidation=None,): self.statusCode = statusCode self.infoMessages = infoMessages self.sqlState = sqlState self.errorCode = errorCode self.errorMessage = errorMessage self.displayMessage = displayMessage + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation def read(self, iprot): @@ -3893,10 +4299,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.infoMessages = [] - (_etype176, _size173) = iprot.readListBegin() - for _i177 in range(_size173): - _elem178 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.infoMessages.append(_elem178) + (_etype201, _size198) = iprot.readListBegin() + for _i202 in range(_size198): + _elem203 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.infoMessages.append(_elem203) iprot.readListEnd() else: iprot.skip(ftype) @@ -3920,6 +4326,11 @@ def read(self, iprot): self.displayMessage = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1281: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -3942,8 +4353,8 @@ def write(self, oprot): if self.infoMessages is not None: oprot.writeFieldBegin('infoMessages', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.infoMessages)) - for iter179 in self.infoMessages: - oprot.writeString(iter179.encode('utf-8') if sys.version_info[0] == 2 else iter179) + for iter204 in self.infoMessages: + oprot.writeString(iter204.encode('utf-8') if sys.version_info[0] == 2 else iter204) oprot.writeListEnd() oprot.writeFieldEnd() if self.sqlState is not None: @@ -3962,6 +4373,10 @@ def write(self, oprot): oprot.writeFieldBegin('displayMessage', TType.STRING, 6) oprot.writeString(self.displayMessage.encode('utf-8') if sys.version_info[0] == 2 else self.displayMessage) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1281) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -4361,21 +4776,21 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype181, _vtype182, _size180) = iprot.readMapBegin() - for _i184 in range(_size180): - _key185 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val186 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key185] = _val186 + (_ktype206, _vtype207, _size205) = iprot.readMapBegin() + for _i209 in range(_size205): + _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key210] = _val211 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype190, _size187) = iprot.readListBegin() - for _i191 in range(_size187): - _elem192 = iprot.readI32() - self.getInfos.append(_elem192) + (_etype215, _size212) = iprot.readListBegin() + for _i216 in range(_size212): + _elem217 = iprot.readI32() + self.getInfos.append(_elem217) iprot.readListEnd() else: iprot.skip(ftype) @@ -4387,11 +4802,11 @@ def read(self, iprot): elif fid == 1283: if ftype == TType.MAP: self.connectionProperties = {} - (_ktype194, _vtype195, _size193) = iprot.readMapBegin() - for _i197 in range(_size193): - _key198 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val199 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.connectionProperties[_key198] = _val199 + (_ktype219, _vtype220, _size218) = iprot.readMapBegin() + for _i222 in range(_size218): + _key223 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val224 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.connectionProperties[_key223] = _val224 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4437,16 +4852,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter200, viter201 in self.configuration.items(): - oprot.writeString(kiter200.encode('utf-8') if sys.version_info[0] == 2 else kiter200) - oprot.writeString(viter201.encode('utf-8') if sys.version_info[0] == 2 else viter201) + for kiter225, viter226 in self.configuration.items(): + oprot.writeString(kiter225.encode('utf-8') if sys.version_info[0] == 2 else kiter225) + oprot.writeString(viter226.encode('utf-8') if sys.version_info[0] == 2 else viter226) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.I32, len(self.getInfos)) - for iter202 in self.getInfos: - oprot.writeI32(iter202) + for iter227 in self.getInfos: + oprot.writeI32(iter227) oprot.writeListEnd() oprot.writeFieldEnd() if self.client_protocol_i64 is not None: @@ -4456,9 +4871,9 @@ def write(self, oprot): if self.connectionProperties is not None: oprot.writeFieldBegin('connectionProperties', TType.MAP, 1283) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.connectionProperties)) - for kiter203, viter204 in self.connectionProperties.items(): - oprot.writeString(kiter203.encode('utf-8') if sys.version_info[0] == 2 else kiter203) - oprot.writeString(viter204.encode('utf-8') if sys.version_info[0] == 2 else viter204) + for kiter228, viter229 in self.connectionProperties.items(): + oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) + oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) oprot.writeMapEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -4543,11 +4958,11 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype206, _vtype207, _size205) = iprot.readMapBegin() - for _i209 in range(_size205): - _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key210] = _val211 + (_ktype231, _vtype232, _size230) = iprot.readMapBegin() + for _i234 in range(_size230): + _key235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val236 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key235] = _val236 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4565,11 +4980,11 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype215, _size212) = iprot.readListBegin() - for _i216 in range(_size212): - _elem217 = TGetInfoValue() - _elem217.read(iprot) - self.getInfos.append(_elem217) + (_etype240, _size237) = iprot.readListBegin() + for _i241 in range(_size237): + _elem242 = TGetInfoValue() + _elem242.read(iprot) + self.getInfos.append(_elem242) iprot.readListEnd() else: iprot.skip(ftype) @@ -4598,16 +5013,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter218, viter219 in self.configuration.items(): - oprot.writeString(kiter218.encode('utf-8') if sys.version_info[0] == 2 else kiter218) - oprot.writeString(viter219.encode('utf-8') if sys.version_info[0] == 2 else viter219) + for kiter243, viter244 in self.configuration.items(): + oprot.writeString(kiter243.encode('utf-8') if sys.version_info[0] == 2 else kiter243) + oprot.writeString(viter244.encode('utf-8') if sys.version_info[0] == 2 else viter244) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.getInfos)) - for iter220 in self.getInfos: - iter220.write(oprot) + for iter245 in self.getInfos: + iter245.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -5202,15 +5617,17 @@ class TSparkArrowTypes(object): - decimalAsArrow - complexTypesAsArrow - intervalTypesAsArrow + - nullTypeAsArrow """ - def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None,): + def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None, nullTypeAsArrow=None,): self.timestampAsArrow = timestampAsArrow self.decimalAsArrow = decimalAsArrow self.complexTypesAsArrow = complexTypesAsArrow self.intervalTypesAsArrow = intervalTypesAsArrow + self.nullTypeAsArrow = nullTypeAsArrow def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5241,6 +5658,11 @@ def read(self, iprot): self.intervalTypesAsArrow = iprot.readBool() else: iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.nullTypeAsArrow = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5267,6 +5689,10 @@ def write(self, oprot): oprot.writeFieldBegin('intervalTypesAsArrow', TType.BOOL, 4) oprot.writeBool(self.intervalTypesAsArrow) oprot.writeFieldEnd() + if self.nullTypeAsArrow is not None: + oprot.writeFieldBegin('nullTypeAsArrow', TType.BOOL, 5) + oprot.writeBool(self.nullTypeAsArrow) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5300,6 +5726,9 @@ class TExecuteStatementReq(object): - maxBytesPerFile - useArrowNativeTypes - resultRowLimit + - parameters + - maxBytesPerBatch + - statementConf - operationId - sessionConf - rejectHighCostQueries @@ -5308,11 +5737,22 @@ class TExecuteStatementReq(object): - requestValidation - resultPersistenceMode - trimArrowBatchesToLimit + - fetchDisposition + - enforceResultPersistenceMode + - statementList + - persistResultManifest + - resultRetentionSeconds + - resultByteLimit + - resultDataFormat + - originatingClientIdentity + - preferSingleFileResult + - preferDriverOnlyUpload + - enforceEmbeddedSchemaCorrectness """ - def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None,): + def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, parameters=None, maxBytesPerBatch=None, statementConf=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None, fetchDisposition=None, enforceResultPersistenceMode=None, statementList=None, persistResultManifest=None, resultRetentionSeconds=None, resultByteLimit=None, resultDataFormat=None, originatingClientIdentity=None, preferSingleFileResult=None, preferDriverOnlyUpload=None, enforceEmbeddedSchemaCorrectness=False,): self.sessionHandle = sessionHandle self.statement = statement self.confOverlay = confOverlay @@ -5325,6 +5765,9 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.maxBytesPerFile = maxBytesPerFile self.useArrowNativeTypes = useArrowNativeTypes self.resultRowLimit = resultRowLimit + self.parameters = parameters + self.maxBytesPerBatch = maxBytesPerBatch + self.statementConf = statementConf self.operationId = operationId self.sessionConf = sessionConf self.rejectHighCostQueries = rejectHighCostQueries @@ -5333,6 +5776,17 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.requestValidation = requestValidation self.resultPersistenceMode = resultPersistenceMode self.trimArrowBatchesToLimit = trimArrowBatchesToLimit + self.fetchDisposition = fetchDisposition + self.enforceResultPersistenceMode = enforceResultPersistenceMode + self.statementList = statementList + self.persistResultManifest = persistResultManifest + self.resultRetentionSeconds = resultRetentionSeconds + self.resultByteLimit = resultByteLimit + self.resultDataFormat = resultDataFormat + self.originatingClientIdentity = originatingClientIdentity + self.preferSingleFileResult = preferSingleFileResult + self.preferDriverOnlyUpload = preferDriverOnlyUpload + self.enforceEmbeddedSchemaCorrectness = enforceEmbeddedSchemaCorrectness def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5357,11 +5811,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.confOverlay = {} - (_ktype222, _vtype223, _size221) = iprot.readMapBegin() - for _i225 in range(_size221): - _key226 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val227 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confOverlay[_key226] = _val227 + (_ktype247, _vtype248, _size246) = iprot.readMapBegin() + for _i250 in range(_size246): + _key251 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val252 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confOverlay[_key251] = _val252 iprot.readMapEnd() else: iprot.skip(ftype) @@ -5412,6 +5866,28 @@ def read(self, iprot): self.resultRowLimit = iprot.readI64() else: iprot.skip(ftype) + elif fid == 1288: + if ftype == TType.LIST: + self.parameters = [] + (_etype256, _size253) = iprot.readListBegin() + for _i257 in range(_size253): + _elem258 = TSparkParameter() + _elem258.read(iprot) + self.parameters.append(_elem258) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1289: + if ftype == TType.I64: + self.maxBytesPerBatch = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 1296: + if ftype == TType.STRUCT: + self.statementConf = TStatementConf() + self.statementConf.read(iprot) + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRUCT: self.operationId = THandleIdentifier() @@ -5454,6 +5930,68 @@ def read(self, iprot): self.trimArrowBatchesToLimit = iprot.readBool() else: iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.BOOL: + self.enforceResultPersistenceMode = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.LIST: + self.statementList = [] + (_etype262, _size259) = iprot.readListBegin() + for _i263 in range(_size259): + _elem264 = TDBSqlStatement() + _elem264.read(iprot) + self.statementList.append(_elem264) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.BOOL: + self.persistResultManifest = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3347: + if ftype == TType.I64: + self.resultRetentionSeconds = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3348: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3349: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3350: + if ftype == TType.STRING: + self.originatingClientIdentity = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3351: + if ftype == TType.BOOL: + self.preferSingleFileResult = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3352: + if ftype == TType.BOOL: + self.preferDriverOnlyUpload = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3353: + if ftype == TType.BOOL: + self.enforceEmbeddedSchemaCorrectness = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5475,9 +6013,9 @@ def write(self, oprot): if self.confOverlay is not None: oprot.writeFieldBegin('confOverlay', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confOverlay)) - for kiter228, viter229 in self.confOverlay.items(): - oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) - oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) + for kiter265, viter266 in self.confOverlay.items(): + oprot.writeString(kiter265.encode('utf-8') if sys.version_info[0] == 2 else kiter265) + oprot.writeString(viter266.encode('utf-8') if sys.version_info[0] == 2 else viter266) oprot.writeMapEnd() oprot.writeFieldEnd() if self.runAsync is not None: @@ -5516,6 +6054,21 @@ def write(self, oprot): oprot.writeFieldBegin('resultRowLimit', TType.I64, 1287) oprot.writeI64(self.resultRowLimit) oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin('parameters', TType.LIST, 1288) + oprot.writeListBegin(TType.STRUCT, len(self.parameters)) + for iter267 in self.parameters: + iter267.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.maxBytesPerBatch is not None: + oprot.writeFieldBegin('maxBytesPerBatch', TType.I64, 1289) + oprot.writeI64(self.maxBytesPerBatch) + oprot.writeFieldEnd() + if self.statementConf is not None: + oprot.writeFieldBegin('statementConf', TType.STRUCT, 1296) + self.statementConf.write(oprot) + oprot.writeFieldEnd() if self.operationId is not None: oprot.writeFieldBegin('operationId', TType.STRUCT, 3329) self.operationId.write(oprot) @@ -5548,6 +6101,53 @@ def write(self, oprot): oprot.writeFieldBegin('trimArrowBatchesToLimit', TType.BOOL, 3336) oprot.writeBool(self.trimArrowBatchesToLimit) oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3337) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.enforceResultPersistenceMode is not None: + oprot.writeFieldBegin('enforceResultPersistenceMode', TType.BOOL, 3344) + oprot.writeBool(self.enforceResultPersistenceMode) + oprot.writeFieldEnd() + if self.statementList is not None: + oprot.writeFieldBegin('statementList', TType.LIST, 3345) + oprot.writeListBegin(TType.STRUCT, len(self.statementList)) + for iter268 in self.statementList: + iter268.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.persistResultManifest is not None: + oprot.writeFieldBegin('persistResultManifest', TType.BOOL, 3346) + oprot.writeBool(self.persistResultManifest) + oprot.writeFieldEnd() + if self.resultRetentionSeconds is not None: + oprot.writeFieldBegin('resultRetentionSeconds', TType.I64, 3347) + oprot.writeI64(self.resultRetentionSeconds) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3348) + oprot.writeI64(self.resultByteLimit) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3349) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.originatingClientIdentity is not None: + oprot.writeFieldBegin('originatingClientIdentity', TType.STRING, 3350) + oprot.writeString(self.originatingClientIdentity.encode('utf-8') if sys.version_info[0] == 2 else self.originatingClientIdentity) + oprot.writeFieldEnd() + if self.preferSingleFileResult is not None: + oprot.writeFieldBegin('preferSingleFileResult', TType.BOOL, 3351) + oprot.writeBool(self.preferSingleFileResult) + oprot.writeFieldEnd() + if self.preferDriverOnlyUpload is not None: + oprot.writeFieldBegin('preferDriverOnlyUpload', TType.BOOL, 3352) + oprot.writeBool(self.preferDriverOnlyUpload) + oprot.writeFieldEnd() + if self.enforceEmbeddedSchemaCorrectness is not None: + oprot.writeFieldBegin('enforceEmbeddedSchemaCorrectness', TType.BOOL, 3353) + oprot.writeBool(self.enforceEmbeddedSchemaCorrectness) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5570,6 +6170,324 @@ def __ne__(self, other): return not (self == other) +class TDBSqlStatement(object): + """ + Attributes: + - statement + + """ + + + def __init__(self, statement=None,): + self.statement = statement + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.statement = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlStatement') + if self.statement is not None: + oprot.writeFieldBegin('statement', TType.STRING, 1) + oprot.writeString(self.statement.encode('utf-8') if sys.version_info[0] == 2 else self.statement) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameterValue(object): + """ + Attributes: + - stringValue + - doubleValue + - booleanValue + + """ + + + def __init__(self, stringValue=None, doubleValue=None, booleanValue=None,): + self.stringValue = stringValue + self.doubleValue = doubleValue + self.booleanValue = booleanValue + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.stringValue = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.doubleValue = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.booleanValue = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameterValue') + if self.stringValue is not None: + oprot.writeFieldBegin('stringValue', TType.STRING, 1) + oprot.writeString(self.stringValue.encode('utf-8') if sys.version_info[0] == 2 else self.stringValue) + oprot.writeFieldEnd() + if self.doubleValue is not None: + oprot.writeFieldBegin('doubleValue', TType.DOUBLE, 2) + oprot.writeDouble(self.doubleValue) + oprot.writeFieldEnd() + if self.booleanValue is not None: + oprot.writeFieldBegin('booleanValue', TType.BOOL, 3) + oprot.writeBool(self.booleanValue) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameter(object): + """ + Attributes: + - ordinal + - name + - type + - value + + """ + + + def __init__(self, ordinal=None, name=None, type=None, value=None,): + self.ordinal = ordinal + self.name = name + self.type = type + self.value = value + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.ordinal = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.type = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.value = TSparkParameterValue() + self.value.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameter') + if self.ordinal is not None: + oprot.writeFieldBegin('ordinal', TType.I32, 1) + oprot.writeI32(self.ordinal) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin('type', TType.STRING, 3) + oprot.writeString(self.type.encode('utf-8') if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRUCT, 4) + self.value.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TStatementConf(object): + """ + Attributes: + - sessionless + - initialNamespace + - client_protocol + - client_protocol_i64 + + """ + + + def __init__(self, sessionless=None, initialNamespace=None, client_protocol=None, client_protocol_i64=None,): + self.sessionless = sessionless + self.initialNamespace = initialNamespace + self.client_protocol = client_protocol + self.client_protocol_i64 = client_protocol_i64 + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.sessionless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.initialNamespace = TNamespace() + self.initialNamespace.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.client_protocol = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.client_protocol_i64 = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TStatementConf') + if self.sessionless is not None: + oprot.writeFieldBegin('sessionless', TType.BOOL, 1) + oprot.writeBool(self.sessionless) + oprot.writeFieldEnd() + if self.initialNamespace is not None: + oprot.writeFieldBegin('initialNamespace', TType.STRUCT, 2) + self.initialNamespace.write(oprot) + oprot.writeFieldEnd() + if self.client_protocol is not None: + oprot.writeFieldBegin('client_protocol', TType.I32, 3) + oprot.writeI32(self.client_protocol) + oprot.writeFieldEnd() + if self.client_protocol_i64 is not None: + oprot.writeFieldBegin('client_protocol_i64', TType.I64, 4) + oprot.writeI64(self.client_protocol_i64) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TExecuteStatementResp(object): """ Attributes: @@ -5582,11 +6500,14 @@ class TExecuteStatementResp(object): - sessionConf - currentClusterLoad - idempotencyType + - remoteResultCacheEnabled + - isServerless + - operationHandles """ - def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None,): + def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None, remoteResultCacheEnabled=None, isServerless=None, operationHandles=None,): self.status = status self.operationHandle = operationHandle self.directResults = directResults @@ -5596,6 +6517,9 @@ def __init__(self, status=None, operationHandle=None, directResults=None, execut self.sessionConf = sessionConf self.currentClusterLoad = currentClusterLoad self.idempotencyType = idempotencyType + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.operationHandles = operationHandles def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5655,6 +6579,27 @@ def read(self, iprot): self.idempotencyType = iprot.readI32() else: iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.LIST: + self.operationHandles = [] + (_etype272, _size269) = iprot.readListBegin() + for _i273 in range(_size269): + _elem274 = TOperationHandle() + _elem274.read(iprot) + self.operationHandles.append(_elem274) + iprot.readListEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5701,6 +6646,21 @@ def write(self, oprot): oprot.writeFieldBegin('idempotencyType', TType.I32, 3334) oprot.writeI32(self.idempotencyType) oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3335) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3336) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.operationHandles is not None: + oprot.writeFieldBegin('operationHandles', TType.LIST, 3337) + oprot.writeListBegin(TType.STRUCT, len(self.operationHandles)) + for iter275 in self.operationHandles: + iter275.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -6376,10 +7336,10 @@ def read(self, iprot): elif fid == 5: if ftype == TType.LIST: self.tableTypes = [] - (_etype233, _size230) = iprot.readListBegin() - for _i234 in range(_size230): - _elem235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.tableTypes.append(_elem235) + (_etype279, _size276) = iprot.readListBegin() + for _i280 in range(_size276): + _elem281 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.tableTypes.append(_elem281) iprot.readListEnd() else: iprot.skip(ftype) @@ -6435,8 +7395,8 @@ def write(self, oprot): if self.tableTypes is not None: oprot.writeFieldBegin('tableTypes', TType.LIST, 5) oprot.writeListBegin(TType.STRING, len(self.tableTypes)) - for iter236 in self.tableTypes: - oprot.writeString(iter236.encode('utf-8') if sys.version_info[0] == 2 else iter236) + for iter282 in self.tableTypes: + oprot.writeString(iter282.encode('utf-8') if sys.version_info[0] == 2 else iter282) oprot.writeListEnd() oprot.writeFieldEnd() if self.getDirectResults is not None: @@ -7783,6 +8743,7 @@ class TGetOperationStatusResp(object): - numModifiedRows - displayMessage - diagnosticInfo + - errorDetailsJson - responseValidation - idempotencyType - statementTimeout @@ -7791,7 +8752,7 @@ class TGetOperationStatusResp(object): """ - def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): + def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, errorDetailsJson=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): self.status = status self.operationState = operationState self.sqlState = sqlState @@ -7805,6 +8766,7 @@ def __init__(self, status=None, operationState=None, sqlState=None, errorCode=No self.numModifiedRows = numModifiedRows self.displayMessage = displayMessage self.diagnosticInfo = diagnosticInfo + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation self.idempotencyType = idempotencyType self.statementTimeout = statementTimeout @@ -7886,6 +8848,11 @@ def read(self, iprot): self.diagnosticInfo = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1283: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -7968,6 +8935,10 @@ def write(self, oprot): oprot.writeFieldBegin('diagnosticInfo', TType.STRING, 1282) oprot.writeString(self.diagnosticInfo.encode('utf-8') if sys.version_info[0] == 2 else self.diagnosticInfo) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1283) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -8353,11 +9324,19 @@ class TGetResultSetMetadataResp(object): - resultFiles - manifestFile - manifestFileFormat + - cacheLookupLatency + - remoteCacheMissReason + - fetchDisposition + - remoteResultCacheEnabled + - isServerless + - resultDataFormat + - truncatedByThriftLimit + - resultByteLimit """ - def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None,): + def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None, cacheLookupLatency=None, remoteCacheMissReason=None, fetchDisposition=None, remoteResultCacheEnabled=None, isServerless=None, resultDataFormat=None, truncatedByThriftLimit=None, resultByteLimit=None,): self.status = status self.schema = schema self.resultFormat = resultFormat @@ -8371,6 +9350,14 @@ def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=No self.resultFiles = resultFiles self.manifestFile = manifestFile self.manifestFileFormat = manifestFileFormat + self.cacheLookupLatency = cacheLookupLatency + self.remoteCacheMissReason = remoteCacheMissReason + self.fetchDisposition = fetchDisposition + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.resultDataFormat = resultDataFormat + self.truncatedByThriftLimit = truncatedByThriftLimit + self.resultByteLimit = resultByteLimit def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -8436,11 +9423,11 @@ def read(self, iprot): elif fid == 3330: if ftype == TType.LIST: self.resultFiles = [] - (_etype240, _size237) = iprot.readListBegin() - for _i241 in range(_size237): - _elem242 = TDBSqlCloudResultFile() - _elem242.read(iprot) - self.resultFiles.append(_elem242) + (_etype286, _size283) = iprot.readListBegin() + for _i287 in range(_size283): + _elem288 = TDBSqlCloudResultFile() + _elem288.read(iprot) + self.resultFiles.append(_elem288) iprot.readListEnd() else: iprot.skip(ftype) @@ -8450,8 +9437,49 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 3332: + if ftype == TType.I32: + self.manifestFileFormat = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3333: + if ftype == TType.I64: + self.cacheLookupLatency = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3334: if ftype == TType.STRING: - self.manifestFileFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.remoteCacheMissReason = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.BOOL: + self.truncatedByThriftLimit = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() else: iprot.skip(ftype) else: @@ -8507,8 +9535,8 @@ def write(self, oprot): if self.resultFiles is not None: oprot.writeFieldBegin('resultFiles', TType.LIST, 3330) oprot.writeListBegin(TType.STRUCT, len(self.resultFiles)) - for iter243 in self.resultFiles: - iter243.write(oprot) + for iter289 in self.resultFiles: + iter289.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.manifestFile is not None: @@ -8516,8 +9544,40 @@ def write(self, oprot): oprot.writeString(self.manifestFile.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFile) oprot.writeFieldEnd() if self.manifestFileFormat is not None: - oprot.writeFieldBegin('manifestFileFormat', TType.STRING, 3332) - oprot.writeString(self.manifestFileFormat.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFileFormat) + oprot.writeFieldBegin('manifestFileFormat', TType.I32, 3332) + oprot.writeI32(self.manifestFileFormat) + oprot.writeFieldEnd() + if self.cacheLookupLatency is not None: + oprot.writeFieldBegin('cacheLookupLatency', TType.I64, 3333) + oprot.writeI64(self.cacheLookupLatency) + oprot.writeFieldEnd() + if self.remoteCacheMissReason is not None: + oprot.writeFieldBegin('remoteCacheMissReason', TType.STRING, 3334) + oprot.writeString(self.remoteCacheMissReason.encode('utf-8') if sys.version_info[0] == 2 else self.remoteCacheMissReason) + oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3335) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3336) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3337) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3344) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.truncatedByThriftLimit is not None: + oprot.writeFieldBegin('truncatedByThriftLimit', TType.BOOL, 3345) + oprot.writeBool(self.truncatedByThriftLimit) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3346) + oprot.writeI64(self.resultByteLimit) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -9267,25 +10327,25 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.headerNames = [] - (_etype247, _size244) = iprot.readListBegin() - for _i248 in range(_size244): - _elem249 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.headerNames.append(_elem249) + (_etype293, _size290) = iprot.readListBegin() + for _i294 in range(_size290): + _elem295 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.headerNames.append(_elem295) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype253, _size250) = iprot.readListBegin() - for _i254 in range(_size250): - _elem255 = [] - (_etype259, _size256) = iprot.readListBegin() - for _i260 in range(_size256): - _elem261 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _elem255.append(_elem261) + (_etype299, _size296) = iprot.readListBegin() + for _i300 in range(_size296): + _elem301 = [] + (_etype305, _size302) = iprot.readListBegin() + for _i306 in range(_size302): + _elem307 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _elem301.append(_elem307) iprot.readListEnd() - self.rows.append(_elem255) + self.rows.append(_elem301) iprot.readListEnd() else: iprot.skip(ftype) @@ -9322,17 +10382,17 @@ def write(self, oprot): if self.headerNames is not None: oprot.writeFieldBegin('headerNames', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.headerNames)) - for iter262 in self.headerNames: - oprot.writeString(iter262.encode('utf-8') if sys.version_info[0] == 2 else iter262) + for iter308 in self.headerNames: + oprot.writeString(iter308.encode('utf-8') if sys.version_info[0] == 2 else iter308) oprot.writeListEnd() oprot.writeFieldEnd() if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.LIST, len(self.rows)) - for iter263 in self.rows: - oprot.writeListBegin(TType.STRING, len(iter263)) - for iter264 in iter263: - oprot.writeString(iter264.encode('utf-8') if sys.version_info[0] == 2 else iter264) + for iter309 in self.rows: + oprot.writeListBegin(TType.STRING, len(iter309)) + for iter310 in iter309: + oprot.writeString(iter310.encode('utf-8') if sys.version_info[0] == 2 else iter310) oprot.writeListEnd() oprot.writeListEnd() oprot.writeFieldEnd() @@ -9380,532 +10440,6 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - -class TDBSqlClusterMetrics(object): - """ - Attributes: - - clusterCapacity - - numRunningTasks - - numPendingTasks - - rejectionThreshold - - tasksCompletedPerMinute - - """ - - - def __init__(self, clusterCapacity=None, numRunningTasks=None, numPendingTasks=None, rejectionThreshold=None, tasksCompletedPerMinute=None,): - self.clusterCapacity = clusterCapacity - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.rejectionThreshold = rejectionThreshold - self.tasksCompletedPerMinute = tasksCompletedPerMinute - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.clusterCapacity = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.DOUBLE: - self.rejectionThreshold = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.DOUBLE: - self.tasksCompletedPerMinute = iprot.readDouble() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlClusterMetrics') - if self.clusterCapacity is not None: - oprot.writeFieldBegin('clusterCapacity', TType.I32, 1) - oprot.writeI32(self.clusterCapacity) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 2) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 3) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.rejectionThreshold is not None: - oprot.writeFieldBegin('rejectionThreshold', TType.DOUBLE, 4) - oprot.writeDouble(self.rejectionThreshold) - oprot.writeFieldEnd() - if self.tasksCompletedPerMinute is not None: - oprot.writeFieldBegin('tasksCompletedPerMinute', TType.DOUBLE, 5) - oprot.writeDouble(self.tasksCompletedPerMinute) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryLaneMetrics(object): - """ - Attributes: - - fastLaneReservation - - numFastLaneRunningTasks - - numFastLanePendingTasks - - slowLaneReservation - - numSlowLaneRunningTasks - - numSlowLanePendingTasks - - """ - - - def __init__(self, fastLaneReservation=None, numFastLaneRunningTasks=None, numFastLanePendingTasks=None, slowLaneReservation=None, numSlowLaneRunningTasks=None, numSlowLanePendingTasks=None,): - self.fastLaneReservation = fastLaneReservation - self.numFastLaneRunningTasks = numFastLaneRunningTasks - self.numFastLanePendingTasks = numFastLanePendingTasks - self.slowLaneReservation = slowLaneReservation - self.numSlowLaneRunningTasks = numSlowLaneRunningTasks - self.numSlowLanePendingTasks = numSlowLanePendingTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.fastLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numFastLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numFastLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.I32: - self.slowLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I32: - self.numSlowLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.I32: - self.numSlowLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryLaneMetrics') - if self.fastLaneReservation is not None: - oprot.writeFieldBegin('fastLaneReservation', TType.I32, 1) - oprot.writeI32(self.fastLaneReservation) - oprot.writeFieldEnd() - if self.numFastLaneRunningTasks is not None: - oprot.writeFieldBegin('numFastLaneRunningTasks', TType.I32, 2) - oprot.writeI32(self.numFastLaneRunningTasks) - oprot.writeFieldEnd() - if self.numFastLanePendingTasks is not None: - oprot.writeFieldBegin('numFastLanePendingTasks', TType.I32, 3) - oprot.writeI32(self.numFastLanePendingTasks) - oprot.writeFieldEnd() - if self.slowLaneReservation is not None: - oprot.writeFieldBegin('slowLaneReservation', TType.I32, 4) - oprot.writeI32(self.slowLaneReservation) - oprot.writeFieldEnd() - if self.numSlowLaneRunningTasks is not None: - oprot.writeFieldBegin('numSlowLaneRunningTasks', TType.I32, 5) - oprot.writeI32(self.numSlowLaneRunningTasks) - oprot.writeFieldEnd() - if self.numSlowLanePendingTasks is not None: - oprot.writeFieldBegin('numSlowLanePendingTasks', TType.I32, 6) - oprot.writeI32(self.numSlowLanePendingTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryMetrics(object): - """ - Attributes: - - status - - operationHandle - - idempotencyType - - sessionHandle - - operationStarted - - queryCost - - numRunningTasks - - numPendingTasks - - numCompletedTasks - - """ - - - def __init__(self, status=None, operationHandle=None, idempotencyType=None, sessionHandle=None, operationStarted=None, queryCost=None, numRunningTasks=None, numPendingTasks=None, numCompletedTasks=None,): - self.status = status - self.operationHandle = operationHandle - self.idempotencyType = idempotencyType - self.sessionHandle = sessionHandle - self.operationStarted = operationStarted - self.queryCost = queryCost - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.numCompletedTasks = numCompletedTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.operationHandle = TOperationHandle() - self.operationHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.idempotencyType = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.sessionHandle = TSessionHandle() - self.sessionHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I64: - self.operationStarted = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.DOUBLE: - self.queryCost = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 7: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 8: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 9: - if ftype == TType.I32: - self.numCompletedTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryMetrics') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.operationHandle is not None: - oprot.writeFieldBegin('operationHandle', TType.STRUCT, 2) - self.operationHandle.write(oprot) - oprot.writeFieldEnd() - if self.idempotencyType is not None: - oprot.writeFieldBegin('idempotencyType', TType.I32, 3) - oprot.writeI32(self.idempotencyType) - oprot.writeFieldEnd() - if self.sessionHandle is not None: - oprot.writeFieldBegin('sessionHandle', TType.STRUCT, 4) - self.sessionHandle.write(oprot) - oprot.writeFieldEnd() - if self.operationStarted is not None: - oprot.writeFieldBegin('operationStarted', TType.I64, 5) - oprot.writeI64(self.operationStarted) - oprot.writeFieldEnd() - if self.queryCost is not None: - oprot.writeFieldBegin('queryCost', TType.DOUBLE, 6) - oprot.writeDouble(self.queryCost) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 7) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 8) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.numCompletedTasks is not None: - oprot.writeFieldBegin('numCompletedTasks', TType.I32, 9) - oprot.writeI32(self.numCompletedTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - if self.operationHandle is None: - raise TProtocolException(message='Required field operationHandle is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationReq(object): - """ - Attributes: - - includeQueryMetrics - - """ - - - def __init__(self, includeQueryMetrics=False,): - self.includeQueryMetrics = includeQueryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.BOOL: - self.includeQueryMetrics = iprot.readBool() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationReq') - if self.includeQueryMetrics is not None: - oprot.writeFieldBegin('includeQueryMetrics', TType.BOOL, 1) - oprot.writeBool(self.includeQueryMetrics) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationResp(object): - """ - Attributes: - - status - - clusterMetrics - - queryLaneMetrics - - queryMetrics - - """ - - - def __init__(self, status=None, clusterMetrics=None, queryLaneMetrics=None, queryMetrics=None,): - self.status = status - self.clusterMetrics = clusterMetrics - self.queryLaneMetrics = queryLaneMetrics - self.queryMetrics = queryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.clusterMetrics = TDBSqlClusterMetrics() - self.clusterMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.queryLaneMetrics = TDBSqlQueryLaneMetrics() - self.queryLaneMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.LIST: - self.queryMetrics = [] - (_etype268, _size265) = iprot.readListBegin() - for _i269 in range(_size265): - _elem270 = TDBSqlQueryMetrics() - _elem270.read(iprot) - self.queryMetrics.append(_elem270) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationResp') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.clusterMetrics is not None: - oprot.writeFieldBegin('clusterMetrics', TType.STRUCT, 2) - self.clusterMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryLaneMetrics is not None: - oprot.writeFieldBegin('queryLaneMetrics', TType.STRUCT, 3) - self.queryLaneMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryMetrics is not None: - oprot.writeFieldBegin('queryMetrics', TType.LIST, 4) - oprot.writeListBegin(TType.STRUCT, len(self.queryMetrics)) - for iter271 in self.queryMetrics: - iter271.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) all_structs.append(TTypeQualifierValue) TTypeQualifierValue.thrift_spec = ( None, # 0 @@ -10088,6 +10622,29 @@ def __ne__(self, other): (7, TType.STRUCT, 'stringVal', [TStringColumn, None], None, ), # 7 (8, TType.STRUCT, 'binaryVal', [TBinaryColumn, None], None, ), # 8 ) +all_structs.append(TDBSqlJsonArrayFormat) +TDBSqlJsonArrayFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlCsvFormat) +TDBSqlCsvFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlArrowFormat) +TDBSqlArrowFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'arrowLayout', None, None, ), # 1 + (2, TType.I32, 'compressionCodec', None, None, ), # 2 +) +all_structs.append(TDBSqlResultFormat) +TDBSqlResultFormat.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'arrowFormat', [TDBSqlArrowFormat, None], None, ), # 1 + (2, TType.STRUCT, 'csvFormat', [TDBSqlCsvFormat, None], None, ), # 2 + (3, TType.STRUCT, 'jsonArrayFormat', [TDBSqlJsonArrayFormat, None], None, ), # 3 +) all_structs.append(TSparkArrowBatch) TSparkArrowBatch.thrift_spec = ( None, # 0 @@ -10102,6 +10659,7 @@ def __ne__(self, other): (3, TType.I64, 'startRowOffset', None, None, ), # 3 (4, TType.I64, 'rowCount', None, None, ), # 4 (5, TType.I64, 'bytesNum', None, None, ), # 5 + (6, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 6 ) all_structs.append(TDBSqlCloudResultFile) TDBSqlCloudResultFile.thrift_spec = ( @@ -10111,6 +10669,9 @@ def __ne__(self, other): (3, TType.I64, 'rowCount', None, None, ), # 3 (4, TType.I64, 'uncompressedBytes', None, None, ), # 4 (5, TType.I64, 'compressedBytes', None, None, ), # 5 + (6, TType.STRING, 'fileLink', 'UTF8', None, ), # 6 + (7, TType.I64, 'linkExpiryTime', None, None, ), # 7 + (8, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 8 ) all_structs.append(TRowSet) TRowSet.thrift_spec = ( @@ -11397,6 +11958,2053 @@ def __ne__(self, other): None, # 1280 (1281, TType.LIST, 'arrowBatches', (TType.STRUCT, [TSparkArrowBatch, None], False), None, ), # 1281 (1282, TType.LIST, 'resultLinks', (TType.STRUCT, [TSparkArrowResultLink, None], False), None, ), # 1282 + None, # 1283 + None, # 1284 + None, # 1285 + None, # 1286 + None, # 1287 + None, # 1288 + None, # 1289 + None, # 1290 + None, # 1291 + None, # 1292 + None, # 1293 + None, # 1294 + None, # 1295 + None, # 1296 + None, # 1297 + None, # 1298 + None, # 1299 + None, # 1300 + None, # 1301 + None, # 1302 + None, # 1303 + None, # 1304 + None, # 1305 + None, # 1306 + None, # 1307 + None, # 1308 + None, # 1309 + None, # 1310 + None, # 1311 + None, # 1312 + None, # 1313 + None, # 1314 + None, # 1315 + None, # 1316 + None, # 1317 + None, # 1318 + None, # 1319 + None, # 1320 + None, # 1321 + None, # 1322 + None, # 1323 + None, # 1324 + None, # 1325 + None, # 1326 + None, # 1327 + None, # 1328 + None, # 1329 + None, # 1330 + None, # 1331 + None, # 1332 + None, # 1333 + None, # 1334 + None, # 1335 + None, # 1336 + None, # 1337 + None, # 1338 + None, # 1339 + None, # 1340 + None, # 1341 + None, # 1342 + None, # 1343 + None, # 1344 + None, # 1345 + None, # 1346 + None, # 1347 + None, # 1348 + None, # 1349 + None, # 1350 + None, # 1351 + None, # 1352 + None, # 1353 + None, # 1354 + None, # 1355 + None, # 1356 + None, # 1357 + None, # 1358 + None, # 1359 + None, # 1360 + None, # 1361 + None, # 1362 + None, # 1363 + None, # 1364 + None, # 1365 + None, # 1366 + None, # 1367 + None, # 1368 + None, # 1369 + None, # 1370 + None, # 1371 + None, # 1372 + None, # 1373 + None, # 1374 + None, # 1375 + None, # 1376 + None, # 1377 + None, # 1378 + None, # 1379 + None, # 1380 + None, # 1381 + None, # 1382 + None, # 1383 + None, # 1384 + None, # 1385 + None, # 1386 + None, # 1387 + None, # 1388 + None, # 1389 + None, # 1390 + None, # 1391 + None, # 1392 + None, # 1393 + None, # 1394 + None, # 1395 + None, # 1396 + None, # 1397 + None, # 1398 + None, # 1399 + None, # 1400 + None, # 1401 + None, # 1402 + None, # 1403 + None, # 1404 + None, # 1405 + None, # 1406 + None, # 1407 + None, # 1408 + None, # 1409 + None, # 1410 + None, # 1411 + None, # 1412 + None, # 1413 + None, # 1414 + None, # 1415 + None, # 1416 + None, # 1417 + None, # 1418 + None, # 1419 + None, # 1420 + None, # 1421 + None, # 1422 + None, # 1423 + None, # 1424 + None, # 1425 + None, # 1426 + None, # 1427 + None, # 1428 + None, # 1429 + None, # 1430 + None, # 1431 + None, # 1432 + None, # 1433 + None, # 1434 + None, # 1435 + None, # 1436 + None, # 1437 + None, # 1438 + None, # 1439 + None, # 1440 + None, # 1441 + None, # 1442 + None, # 1443 + None, # 1444 + None, # 1445 + None, # 1446 + None, # 1447 + None, # 1448 + None, # 1449 + None, # 1450 + None, # 1451 + None, # 1452 + None, # 1453 + None, # 1454 + None, # 1455 + None, # 1456 + None, # 1457 + None, # 1458 + None, # 1459 + None, # 1460 + None, # 1461 + None, # 1462 + None, # 1463 + None, # 1464 + None, # 1465 + None, # 1466 + None, # 1467 + None, # 1468 + None, # 1469 + None, # 1470 + None, # 1471 + None, # 1472 + None, # 1473 + None, # 1474 + None, # 1475 + None, # 1476 + None, # 1477 + None, # 1478 + None, # 1479 + None, # 1480 + None, # 1481 + None, # 1482 + None, # 1483 + None, # 1484 + None, # 1485 + None, # 1486 + None, # 1487 + None, # 1488 + None, # 1489 + None, # 1490 + None, # 1491 + None, # 1492 + None, # 1493 + None, # 1494 + None, # 1495 + None, # 1496 + None, # 1497 + None, # 1498 + None, # 1499 + None, # 1500 + None, # 1501 + None, # 1502 + None, # 1503 + None, # 1504 + None, # 1505 + None, # 1506 + None, # 1507 + None, # 1508 + None, # 1509 + None, # 1510 + None, # 1511 + None, # 1512 + None, # 1513 + None, # 1514 + None, # 1515 + None, # 1516 + None, # 1517 + None, # 1518 + None, # 1519 + None, # 1520 + None, # 1521 + None, # 1522 + None, # 1523 + None, # 1524 + None, # 1525 + None, # 1526 + None, # 1527 + None, # 1528 + None, # 1529 + None, # 1530 + None, # 1531 + None, # 1532 + None, # 1533 + None, # 1534 + None, # 1535 + None, # 1536 + None, # 1537 + None, # 1538 + None, # 1539 + None, # 1540 + None, # 1541 + None, # 1542 + None, # 1543 + None, # 1544 + None, # 1545 + None, # 1546 + None, # 1547 + None, # 1548 + None, # 1549 + None, # 1550 + None, # 1551 + None, # 1552 + None, # 1553 + None, # 1554 + None, # 1555 + None, # 1556 + None, # 1557 + None, # 1558 + None, # 1559 + None, # 1560 + None, # 1561 + None, # 1562 + None, # 1563 + None, # 1564 + None, # 1565 + None, # 1566 + None, # 1567 + None, # 1568 + None, # 1569 + None, # 1570 + None, # 1571 + None, # 1572 + None, # 1573 + None, # 1574 + None, # 1575 + None, # 1576 + None, # 1577 + None, # 1578 + None, # 1579 + None, # 1580 + None, # 1581 + None, # 1582 + None, # 1583 + None, # 1584 + None, # 1585 + None, # 1586 + None, # 1587 + None, # 1588 + None, # 1589 + None, # 1590 + None, # 1591 + None, # 1592 + None, # 1593 + None, # 1594 + None, # 1595 + None, # 1596 + None, # 1597 + None, # 1598 + None, # 1599 + None, # 1600 + None, # 1601 + None, # 1602 + None, # 1603 + None, # 1604 + None, # 1605 + None, # 1606 + None, # 1607 + None, # 1608 + None, # 1609 + None, # 1610 + None, # 1611 + None, # 1612 + None, # 1613 + None, # 1614 + None, # 1615 + None, # 1616 + None, # 1617 + None, # 1618 + None, # 1619 + None, # 1620 + None, # 1621 + None, # 1622 + None, # 1623 + None, # 1624 + None, # 1625 + None, # 1626 + None, # 1627 + None, # 1628 + None, # 1629 + None, # 1630 + None, # 1631 + None, # 1632 + None, # 1633 + None, # 1634 + None, # 1635 + None, # 1636 + None, # 1637 + None, # 1638 + None, # 1639 + None, # 1640 + None, # 1641 + None, # 1642 + None, # 1643 + None, # 1644 + None, # 1645 + None, # 1646 + None, # 1647 + None, # 1648 + None, # 1649 + None, # 1650 + None, # 1651 + None, # 1652 + None, # 1653 + None, # 1654 + None, # 1655 + None, # 1656 + None, # 1657 + None, # 1658 + None, # 1659 + None, # 1660 + None, # 1661 + None, # 1662 + None, # 1663 + None, # 1664 + None, # 1665 + None, # 1666 + None, # 1667 + None, # 1668 + None, # 1669 + None, # 1670 + None, # 1671 + None, # 1672 + None, # 1673 + None, # 1674 + None, # 1675 + None, # 1676 + None, # 1677 + None, # 1678 + None, # 1679 + None, # 1680 + None, # 1681 + None, # 1682 + None, # 1683 + None, # 1684 + None, # 1685 + None, # 1686 + None, # 1687 + None, # 1688 + None, # 1689 + None, # 1690 + None, # 1691 + None, # 1692 + None, # 1693 + None, # 1694 + None, # 1695 + None, # 1696 + None, # 1697 + None, # 1698 + None, # 1699 + None, # 1700 + None, # 1701 + None, # 1702 + None, # 1703 + None, # 1704 + None, # 1705 + None, # 1706 + None, # 1707 + None, # 1708 + None, # 1709 + None, # 1710 + None, # 1711 + None, # 1712 + None, # 1713 + None, # 1714 + None, # 1715 + None, # 1716 + None, # 1717 + None, # 1718 + None, # 1719 + None, # 1720 + None, # 1721 + None, # 1722 + None, # 1723 + None, # 1724 + None, # 1725 + None, # 1726 + None, # 1727 + None, # 1728 + None, # 1729 + None, # 1730 + None, # 1731 + None, # 1732 + None, # 1733 + None, # 1734 + None, # 1735 + None, # 1736 + None, # 1737 + None, # 1738 + None, # 1739 + None, # 1740 + None, # 1741 + None, # 1742 + None, # 1743 + None, # 1744 + None, # 1745 + None, # 1746 + None, # 1747 + None, # 1748 + None, # 1749 + None, # 1750 + None, # 1751 + None, # 1752 + None, # 1753 + None, # 1754 + None, # 1755 + None, # 1756 + None, # 1757 + None, # 1758 + None, # 1759 + None, # 1760 + None, # 1761 + None, # 1762 + None, # 1763 + None, # 1764 + None, # 1765 + None, # 1766 + None, # 1767 + None, # 1768 + None, # 1769 + None, # 1770 + None, # 1771 + None, # 1772 + None, # 1773 + None, # 1774 + None, # 1775 + None, # 1776 + None, # 1777 + None, # 1778 + None, # 1779 + None, # 1780 + None, # 1781 + None, # 1782 + None, # 1783 + None, # 1784 + None, # 1785 + None, # 1786 + None, # 1787 + None, # 1788 + None, # 1789 + None, # 1790 + None, # 1791 + None, # 1792 + None, # 1793 + None, # 1794 + None, # 1795 + None, # 1796 + None, # 1797 + None, # 1798 + None, # 1799 + None, # 1800 + None, # 1801 + None, # 1802 + None, # 1803 + None, # 1804 + None, # 1805 + None, # 1806 + None, # 1807 + None, # 1808 + None, # 1809 + None, # 1810 + None, # 1811 + None, # 1812 + None, # 1813 + None, # 1814 + None, # 1815 + None, # 1816 + None, # 1817 + None, # 1818 + None, # 1819 + None, # 1820 + None, # 1821 + None, # 1822 + None, # 1823 + None, # 1824 + None, # 1825 + None, # 1826 + None, # 1827 + None, # 1828 + None, # 1829 + None, # 1830 + None, # 1831 + None, # 1832 + None, # 1833 + None, # 1834 + None, # 1835 + None, # 1836 + None, # 1837 + None, # 1838 + None, # 1839 + None, # 1840 + None, # 1841 + None, # 1842 + None, # 1843 + None, # 1844 + None, # 1845 + None, # 1846 + None, # 1847 + None, # 1848 + None, # 1849 + None, # 1850 + None, # 1851 + None, # 1852 + None, # 1853 + None, # 1854 + None, # 1855 + None, # 1856 + None, # 1857 + None, # 1858 + None, # 1859 + None, # 1860 + None, # 1861 + None, # 1862 + None, # 1863 + None, # 1864 + None, # 1865 + None, # 1866 + None, # 1867 + None, # 1868 + None, # 1869 + None, # 1870 + None, # 1871 + None, # 1872 + None, # 1873 + None, # 1874 + None, # 1875 + None, # 1876 + None, # 1877 + None, # 1878 + None, # 1879 + None, # 1880 + None, # 1881 + None, # 1882 + None, # 1883 + None, # 1884 + None, # 1885 + None, # 1886 + None, # 1887 + None, # 1888 + None, # 1889 + None, # 1890 + None, # 1891 + None, # 1892 + None, # 1893 + None, # 1894 + None, # 1895 + None, # 1896 + None, # 1897 + None, # 1898 + None, # 1899 + None, # 1900 + None, # 1901 + None, # 1902 + None, # 1903 + None, # 1904 + None, # 1905 + None, # 1906 + None, # 1907 + None, # 1908 + None, # 1909 + None, # 1910 + None, # 1911 + None, # 1912 + None, # 1913 + None, # 1914 + None, # 1915 + None, # 1916 + None, # 1917 + None, # 1918 + None, # 1919 + None, # 1920 + None, # 1921 + None, # 1922 + None, # 1923 + None, # 1924 + None, # 1925 + None, # 1926 + None, # 1927 + None, # 1928 + None, # 1929 + None, # 1930 + None, # 1931 + None, # 1932 + None, # 1933 + None, # 1934 + None, # 1935 + None, # 1936 + None, # 1937 + None, # 1938 + None, # 1939 + None, # 1940 + None, # 1941 + None, # 1942 + None, # 1943 + None, # 1944 + None, # 1945 + None, # 1946 + None, # 1947 + None, # 1948 + None, # 1949 + None, # 1950 + None, # 1951 + None, # 1952 + None, # 1953 + None, # 1954 + None, # 1955 + None, # 1956 + None, # 1957 + None, # 1958 + None, # 1959 + None, # 1960 + None, # 1961 + None, # 1962 + None, # 1963 + None, # 1964 + None, # 1965 + None, # 1966 + None, # 1967 + None, # 1968 + None, # 1969 + None, # 1970 + None, # 1971 + None, # 1972 + None, # 1973 + None, # 1974 + None, # 1975 + None, # 1976 + None, # 1977 + None, # 1978 + None, # 1979 + None, # 1980 + None, # 1981 + None, # 1982 + None, # 1983 + None, # 1984 + None, # 1985 + None, # 1986 + None, # 1987 + None, # 1988 + None, # 1989 + None, # 1990 + None, # 1991 + None, # 1992 + None, # 1993 + None, # 1994 + None, # 1995 + None, # 1996 + None, # 1997 + None, # 1998 + None, # 1999 + None, # 2000 + None, # 2001 + None, # 2002 + None, # 2003 + None, # 2004 + None, # 2005 + None, # 2006 + None, # 2007 + None, # 2008 + None, # 2009 + None, # 2010 + None, # 2011 + None, # 2012 + None, # 2013 + None, # 2014 + None, # 2015 + None, # 2016 + None, # 2017 + None, # 2018 + None, # 2019 + None, # 2020 + None, # 2021 + None, # 2022 + None, # 2023 + None, # 2024 + None, # 2025 + None, # 2026 + None, # 2027 + None, # 2028 + None, # 2029 + None, # 2030 + None, # 2031 + None, # 2032 + None, # 2033 + None, # 2034 + None, # 2035 + None, # 2036 + None, # 2037 + None, # 2038 + None, # 2039 + None, # 2040 + None, # 2041 + None, # 2042 + None, # 2043 + None, # 2044 + None, # 2045 + None, # 2046 + None, # 2047 + None, # 2048 + None, # 2049 + None, # 2050 + None, # 2051 + None, # 2052 + None, # 2053 + None, # 2054 + None, # 2055 + None, # 2056 + None, # 2057 + None, # 2058 + None, # 2059 + None, # 2060 + None, # 2061 + None, # 2062 + None, # 2063 + None, # 2064 + None, # 2065 + None, # 2066 + None, # 2067 + None, # 2068 + None, # 2069 + None, # 2070 + None, # 2071 + None, # 2072 + None, # 2073 + None, # 2074 + None, # 2075 + None, # 2076 + None, # 2077 + None, # 2078 + None, # 2079 + None, # 2080 + None, # 2081 + None, # 2082 + None, # 2083 + None, # 2084 + None, # 2085 + None, # 2086 + None, # 2087 + None, # 2088 + None, # 2089 + None, # 2090 + None, # 2091 + None, # 2092 + None, # 2093 + None, # 2094 + None, # 2095 + None, # 2096 + None, # 2097 + None, # 2098 + None, # 2099 + None, # 2100 + None, # 2101 + None, # 2102 + None, # 2103 + None, # 2104 + None, # 2105 + None, # 2106 + None, # 2107 + None, # 2108 + None, # 2109 + None, # 2110 + None, # 2111 + None, # 2112 + None, # 2113 + None, # 2114 + None, # 2115 + None, # 2116 + None, # 2117 + None, # 2118 + None, # 2119 + None, # 2120 + None, # 2121 + None, # 2122 + None, # 2123 + None, # 2124 + None, # 2125 + None, # 2126 + None, # 2127 + None, # 2128 + None, # 2129 + None, # 2130 + None, # 2131 + None, # 2132 + None, # 2133 + None, # 2134 + None, # 2135 + None, # 2136 + None, # 2137 + None, # 2138 + None, # 2139 + None, # 2140 + None, # 2141 + None, # 2142 + None, # 2143 + None, # 2144 + None, # 2145 + None, # 2146 + None, # 2147 + None, # 2148 + None, # 2149 + None, # 2150 + None, # 2151 + None, # 2152 + None, # 2153 + None, # 2154 + None, # 2155 + None, # 2156 + None, # 2157 + None, # 2158 + None, # 2159 + None, # 2160 + None, # 2161 + None, # 2162 + None, # 2163 + None, # 2164 + None, # 2165 + None, # 2166 + None, # 2167 + None, # 2168 + None, # 2169 + None, # 2170 + None, # 2171 + None, # 2172 + None, # 2173 + None, # 2174 + None, # 2175 + None, # 2176 + None, # 2177 + None, # 2178 + None, # 2179 + None, # 2180 + None, # 2181 + None, # 2182 + None, # 2183 + None, # 2184 + None, # 2185 + None, # 2186 + None, # 2187 + None, # 2188 + None, # 2189 + None, # 2190 + None, # 2191 + None, # 2192 + None, # 2193 + None, # 2194 + None, # 2195 + None, # 2196 + None, # 2197 + None, # 2198 + None, # 2199 + None, # 2200 + None, # 2201 + None, # 2202 + None, # 2203 + None, # 2204 + None, # 2205 + None, # 2206 + None, # 2207 + None, # 2208 + None, # 2209 + None, # 2210 + None, # 2211 + None, # 2212 + None, # 2213 + None, # 2214 + None, # 2215 + None, # 2216 + None, # 2217 + None, # 2218 + None, # 2219 + None, # 2220 + None, # 2221 + None, # 2222 + None, # 2223 + None, # 2224 + None, # 2225 + None, # 2226 + None, # 2227 + None, # 2228 + None, # 2229 + None, # 2230 + None, # 2231 + None, # 2232 + None, # 2233 + None, # 2234 + None, # 2235 + None, # 2236 + None, # 2237 + None, # 2238 + None, # 2239 + None, # 2240 + None, # 2241 + None, # 2242 + None, # 2243 + None, # 2244 + None, # 2245 + None, # 2246 + None, # 2247 + None, # 2248 + None, # 2249 + None, # 2250 + None, # 2251 + None, # 2252 + None, # 2253 + None, # 2254 + None, # 2255 + None, # 2256 + None, # 2257 + None, # 2258 + None, # 2259 + None, # 2260 + None, # 2261 + None, # 2262 + None, # 2263 + None, # 2264 + None, # 2265 + None, # 2266 + None, # 2267 + None, # 2268 + None, # 2269 + None, # 2270 + None, # 2271 + None, # 2272 + None, # 2273 + None, # 2274 + None, # 2275 + None, # 2276 + None, # 2277 + None, # 2278 + None, # 2279 + None, # 2280 + None, # 2281 + None, # 2282 + None, # 2283 + None, # 2284 + None, # 2285 + None, # 2286 + None, # 2287 + None, # 2288 + None, # 2289 + None, # 2290 + None, # 2291 + None, # 2292 + None, # 2293 + None, # 2294 + None, # 2295 + None, # 2296 + None, # 2297 + None, # 2298 + None, # 2299 + None, # 2300 + None, # 2301 + None, # 2302 + None, # 2303 + None, # 2304 + None, # 2305 + None, # 2306 + None, # 2307 + None, # 2308 + None, # 2309 + None, # 2310 + None, # 2311 + None, # 2312 + None, # 2313 + None, # 2314 + None, # 2315 + None, # 2316 + None, # 2317 + None, # 2318 + None, # 2319 + None, # 2320 + None, # 2321 + None, # 2322 + None, # 2323 + None, # 2324 + None, # 2325 + None, # 2326 + None, # 2327 + None, # 2328 + None, # 2329 + None, # 2330 + None, # 2331 + None, # 2332 + None, # 2333 + None, # 2334 + None, # 2335 + None, # 2336 + None, # 2337 + None, # 2338 + None, # 2339 + None, # 2340 + None, # 2341 + None, # 2342 + None, # 2343 + None, # 2344 + None, # 2345 + None, # 2346 + None, # 2347 + None, # 2348 + None, # 2349 + None, # 2350 + None, # 2351 + None, # 2352 + None, # 2353 + None, # 2354 + None, # 2355 + None, # 2356 + None, # 2357 + None, # 2358 + None, # 2359 + None, # 2360 + None, # 2361 + None, # 2362 + None, # 2363 + None, # 2364 + None, # 2365 + None, # 2366 + None, # 2367 + None, # 2368 + None, # 2369 + None, # 2370 + None, # 2371 + None, # 2372 + None, # 2373 + None, # 2374 + None, # 2375 + None, # 2376 + None, # 2377 + None, # 2378 + None, # 2379 + None, # 2380 + None, # 2381 + None, # 2382 + None, # 2383 + None, # 2384 + None, # 2385 + None, # 2386 + None, # 2387 + None, # 2388 + None, # 2389 + None, # 2390 + None, # 2391 + None, # 2392 + None, # 2393 + None, # 2394 + None, # 2395 + None, # 2396 + None, # 2397 + None, # 2398 + None, # 2399 + None, # 2400 + None, # 2401 + None, # 2402 + None, # 2403 + None, # 2404 + None, # 2405 + None, # 2406 + None, # 2407 + None, # 2408 + None, # 2409 + None, # 2410 + None, # 2411 + None, # 2412 + None, # 2413 + None, # 2414 + None, # 2415 + None, # 2416 + None, # 2417 + None, # 2418 + None, # 2419 + None, # 2420 + None, # 2421 + None, # 2422 + None, # 2423 + None, # 2424 + None, # 2425 + None, # 2426 + None, # 2427 + None, # 2428 + None, # 2429 + None, # 2430 + None, # 2431 + None, # 2432 + None, # 2433 + None, # 2434 + None, # 2435 + None, # 2436 + None, # 2437 + None, # 2438 + None, # 2439 + None, # 2440 + None, # 2441 + None, # 2442 + None, # 2443 + None, # 2444 + None, # 2445 + None, # 2446 + None, # 2447 + None, # 2448 + None, # 2449 + None, # 2450 + None, # 2451 + None, # 2452 + None, # 2453 + None, # 2454 + None, # 2455 + None, # 2456 + None, # 2457 + None, # 2458 + None, # 2459 + None, # 2460 + None, # 2461 + None, # 2462 + None, # 2463 + None, # 2464 + None, # 2465 + None, # 2466 + None, # 2467 + None, # 2468 + None, # 2469 + None, # 2470 + None, # 2471 + None, # 2472 + None, # 2473 + None, # 2474 + None, # 2475 + None, # 2476 + None, # 2477 + None, # 2478 + None, # 2479 + None, # 2480 + None, # 2481 + None, # 2482 + None, # 2483 + None, # 2484 + None, # 2485 + None, # 2486 + None, # 2487 + None, # 2488 + None, # 2489 + None, # 2490 + None, # 2491 + None, # 2492 + None, # 2493 + None, # 2494 + None, # 2495 + None, # 2496 + None, # 2497 + None, # 2498 + None, # 2499 + None, # 2500 + None, # 2501 + None, # 2502 + None, # 2503 + None, # 2504 + None, # 2505 + None, # 2506 + None, # 2507 + None, # 2508 + None, # 2509 + None, # 2510 + None, # 2511 + None, # 2512 + None, # 2513 + None, # 2514 + None, # 2515 + None, # 2516 + None, # 2517 + None, # 2518 + None, # 2519 + None, # 2520 + None, # 2521 + None, # 2522 + None, # 2523 + None, # 2524 + None, # 2525 + None, # 2526 + None, # 2527 + None, # 2528 + None, # 2529 + None, # 2530 + None, # 2531 + None, # 2532 + None, # 2533 + None, # 2534 + None, # 2535 + None, # 2536 + None, # 2537 + None, # 2538 + None, # 2539 + None, # 2540 + None, # 2541 + None, # 2542 + None, # 2543 + None, # 2544 + None, # 2545 + None, # 2546 + None, # 2547 + None, # 2548 + None, # 2549 + None, # 2550 + None, # 2551 + None, # 2552 + None, # 2553 + None, # 2554 + None, # 2555 + None, # 2556 + None, # 2557 + None, # 2558 + None, # 2559 + None, # 2560 + None, # 2561 + None, # 2562 + None, # 2563 + None, # 2564 + None, # 2565 + None, # 2566 + None, # 2567 + None, # 2568 + None, # 2569 + None, # 2570 + None, # 2571 + None, # 2572 + None, # 2573 + None, # 2574 + None, # 2575 + None, # 2576 + None, # 2577 + None, # 2578 + None, # 2579 + None, # 2580 + None, # 2581 + None, # 2582 + None, # 2583 + None, # 2584 + None, # 2585 + None, # 2586 + None, # 2587 + None, # 2588 + None, # 2589 + None, # 2590 + None, # 2591 + None, # 2592 + None, # 2593 + None, # 2594 + None, # 2595 + None, # 2596 + None, # 2597 + None, # 2598 + None, # 2599 + None, # 2600 + None, # 2601 + None, # 2602 + None, # 2603 + None, # 2604 + None, # 2605 + None, # 2606 + None, # 2607 + None, # 2608 + None, # 2609 + None, # 2610 + None, # 2611 + None, # 2612 + None, # 2613 + None, # 2614 + None, # 2615 + None, # 2616 + None, # 2617 + None, # 2618 + None, # 2619 + None, # 2620 + None, # 2621 + None, # 2622 + None, # 2623 + None, # 2624 + None, # 2625 + None, # 2626 + None, # 2627 + None, # 2628 + None, # 2629 + None, # 2630 + None, # 2631 + None, # 2632 + None, # 2633 + None, # 2634 + None, # 2635 + None, # 2636 + None, # 2637 + None, # 2638 + None, # 2639 + None, # 2640 + None, # 2641 + None, # 2642 + None, # 2643 + None, # 2644 + None, # 2645 + None, # 2646 + None, # 2647 + None, # 2648 + None, # 2649 + None, # 2650 + None, # 2651 + None, # 2652 + None, # 2653 + None, # 2654 + None, # 2655 + None, # 2656 + None, # 2657 + None, # 2658 + None, # 2659 + None, # 2660 + None, # 2661 + None, # 2662 + None, # 2663 + None, # 2664 + None, # 2665 + None, # 2666 + None, # 2667 + None, # 2668 + None, # 2669 + None, # 2670 + None, # 2671 + None, # 2672 + None, # 2673 + None, # 2674 + None, # 2675 + None, # 2676 + None, # 2677 + None, # 2678 + None, # 2679 + None, # 2680 + None, # 2681 + None, # 2682 + None, # 2683 + None, # 2684 + None, # 2685 + None, # 2686 + None, # 2687 + None, # 2688 + None, # 2689 + None, # 2690 + None, # 2691 + None, # 2692 + None, # 2693 + None, # 2694 + None, # 2695 + None, # 2696 + None, # 2697 + None, # 2698 + None, # 2699 + None, # 2700 + None, # 2701 + None, # 2702 + None, # 2703 + None, # 2704 + None, # 2705 + None, # 2706 + None, # 2707 + None, # 2708 + None, # 2709 + None, # 2710 + None, # 2711 + None, # 2712 + None, # 2713 + None, # 2714 + None, # 2715 + None, # 2716 + None, # 2717 + None, # 2718 + None, # 2719 + None, # 2720 + None, # 2721 + None, # 2722 + None, # 2723 + None, # 2724 + None, # 2725 + None, # 2726 + None, # 2727 + None, # 2728 + None, # 2729 + None, # 2730 + None, # 2731 + None, # 2732 + None, # 2733 + None, # 2734 + None, # 2735 + None, # 2736 + None, # 2737 + None, # 2738 + None, # 2739 + None, # 2740 + None, # 2741 + None, # 2742 + None, # 2743 + None, # 2744 + None, # 2745 + None, # 2746 + None, # 2747 + None, # 2748 + None, # 2749 + None, # 2750 + None, # 2751 + None, # 2752 + None, # 2753 + None, # 2754 + None, # 2755 + None, # 2756 + None, # 2757 + None, # 2758 + None, # 2759 + None, # 2760 + None, # 2761 + None, # 2762 + None, # 2763 + None, # 2764 + None, # 2765 + None, # 2766 + None, # 2767 + None, # 2768 + None, # 2769 + None, # 2770 + None, # 2771 + None, # 2772 + None, # 2773 + None, # 2774 + None, # 2775 + None, # 2776 + None, # 2777 + None, # 2778 + None, # 2779 + None, # 2780 + None, # 2781 + None, # 2782 + None, # 2783 + None, # 2784 + None, # 2785 + None, # 2786 + None, # 2787 + None, # 2788 + None, # 2789 + None, # 2790 + None, # 2791 + None, # 2792 + None, # 2793 + None, # 2794 + None, # 2795 + None, # 2796 + None, # 2797 + None, # 2798 + None, # 2799 + None, # 2800 + None, # 2801 + None, # 2802 + None, # 2803 + None, # 2804 + None, # 2805 + None, # 2806 + None, # 2807 + None, # 2808 + None, # 2809 + None, # 2810 + None, # 2811 + None, # 2812 + None, # 2813 + None, # 2814 + None, # 2815 + None, # 2816 + None, # 2817 + None, # 2818 + None, # 2819 + None, # 2820 + None, # 2821 + None, # 2822 + None, # 2823 + None, # 2824 + None, # 2825 + None, # 2826 + None, # 2827 + None, # 2828 + None, # 2829 + None, # 2830 + None, # 2831 + None, # 2832 + None, # 2833 + None, # 2834 + None, # 2835 + None, # 2836 + None, # 2837 + None, # 2838 + None, # 2839 + None, # 2840 + None, # 2841 + None, # 2842 + None, # 2843 + None, # 2844 + None, # 2845 + None, # 2846 + None, # 2847 + None, # 2848 + None, # 2849 + None, # 2850 + None, # 2851 + None, # 2852 + None, # 2853 + None, # 2854 + None, # 2855 + None, # 2856 + None, # 2857 + None, # 2858 + None, # 2859 + None, # 2860 + None, # 2861 + None, # 2862 + None, # 2863 + None, # 2864 + None, # 2865 + None, # 2866 + None, # 2867 + None, # 2868 + None, # 2869 + None, # 2870 + None, # 2871 + None, # 2872 + None, # 2873 + None, # 2874 + None, # 2875 + None, # 2876 + None, # 2877 + None, # 2878 + None, # 2879 + None, # 2880 + None, # 2881 + None, # 2882 + None, # 2883 + None, # 2884 + None, # 2885 + None, # 2886 + None, # 2887 + None, # 2888 + None, # 2889 + None, # 2890 + None, # 2891 + None, # 2892 + None, # 2893 + None, # 2894 + None, # 2895 + None, # 2896 + None, # 2897 + None, # 2898 + None, # 2899 + None, # 2900 + None, # 2901 + None, # 2902 + None, # 2903 + None, # 2904 + None, # 2905 + None, # 2906 + None, # 2907 + None, # 2908 + None, # 2909 + None, # 2910 + None, # 2911 + None, # 2912 + None, # 2913 + None, # 2914 + None, # 2915 + None, # 2916 + None, # 2917 + None, # 2918 + None, # 2919 + None, # 2920 + None, # 2921 + None, # 2922 + None, # 2923 + None, # 2924 + None, # 2925 + None, # 2926 + None, # 2927 + None, # 2928 + None, # 2929 + None, # 2930 + None, # 2931 + None, # 2932 + None, # 2933 + None, # 2934 + None, # 2935 + None, # 2936 + None, # 2937 + None, # 2938 + None, # 2939 + None, # 2940 + None, # 2941 + None, # 2942 + None, # 2943 + None, # 2944 + None, # 2945 + None, # 2946 + None, # 2947 + None, # 2948 + None, # 2949 + None, # 2950 + None, # 2951 + None, # 2952 + None, # 2953 + None, # 2954 + None, # 2955 + None, # 2956 + None, # 2957 + None, # 2958 + None, # 2959 + None, # 2960 + None, # 2961 + None, # 2962 + None, # 2963 + None, # 2964 + None, # 2965 + None, # 2966 + None, # 2967 + None, # 2968 + None, # 2969 + None, # 2970 + None, # 2971 + None, # 2972 + None, # 2973 + None, # 2974 + None, # 2975 + None, # 2976 + None, # 2977 + None, # 2978 + None, # 2979 + None, # 2980 + None, # 2981 + None, # 2982 + None, # 2983 + None, # 2984 + None, # 2985 + None, # 2986 + None, # 2987 + None, # 2988 + None, # 2989 + None, # 2990 + None, # 2991 + None, # 2992 + None, # 2993 + None, # 2994 + None, # 2995 + None, # 2996 + None, # 2997 + None, # 2998 + None, # 2999 + None, # 3000 + None, # 3001 + None, # 3002 + None, # 3003 + None, # 3004 + None, # 3005 + None, # 3006 + None, # 3007 + None, # 3008 + None, # 3009 + None, # 3010 + None, # 3011 + None, # 3012 + None, # 3013 + None, # 3014 + None, # 3015 + None, # 3016 + None, # 3017 + None, # 3018 + None, # 3019 + None, # 3020 + None, # 3021 + None, # 3022 + None, # 3023 + None, # 3024 + None, # 3025 + None, # 3026 + None, # 3027 + None, # 3028 + None, # 3029 + None, # 3030 + None, # 3031 + None, # 3032 + None, # 3033 + None, # 3034 + None, # 3035 + None, # 3036 + None, # 3037 + None, # 3038 + None, # 3039 + None, # 3040 + None, # 3041 + None, # 3042 + None, # 3043 + None, # 3044 + None, # 3045 + None, # 3046 + None, # 3047 + None, # 3048 + None, # 3049 + None, # 3050 + None, # 3051 + None, # 3052 + None, # 3053 + None, # 3054 + None, # 3055 + None, # 3056 + None, # 3057 + None, # 3058 + None, # 3059 + None, # 3060 + None, # 3061 + None, # 3062 + None, # 3063 + None, # 3064 + None, # 3065 + None, # 3066 + None, # 3067 + None, # 3068 + None, # 3069 + None, # 3070 + None, # 3071 + None, # 3072 + None, # 3073 + None, # 3074 + None, # 3075 + None, # 3076 + None, # 3077 + None, # 3078 + None, # 3079 + None, # 3080 + None, # 3081 + None, # 3082 + None, # 3083 + None, # 3084 + None, # 3085 + None, # 3086 + None, # 3087 + None, # 3088 + None, # 3089 + None, # 3090 + None, # 3091 + None, # 3092 + None, # 3093 + None, # 3094 + None, # 3095 + None, # 3096 + None, # 3097 + None, # 3098 + None, # 3099 + None, # 3100 + None, # 3101 + None, # 3102 + None, # 3103 + None, # 3104 + None, # 3105 + None, # 3106 + None, # 3107 + None, # 3108 + None, # 3109 + None, # 3110 + None, # 3111 + None, # 3112 + None, # 3113 + None, # 3114 + None, # 3115 + None, # 3116 + None, # 3117 + None, # 3118 + None, # 3119 + None, # 3120 + None, # 3121 + None, # 3122 + None, # 3123 + None, # 3124 + None, # 3125 + None, # 3126 + None, # 3127 + None, # 3128 + None, # 3129 + None, # 3130 + None, # 3131 + None, # 3132 + None, # 3133 + None, # 3134 + None, # 3135 + None, # 3136 + None, # 3137 + None, # 3138 + None, # 3139 + None, # 3140 + None, # 3141 + None, # 3142 + None, # 3143 + None, # 3144 + None, # 3145 + None, # 3146 + None, # 3147 + None, # 3148 + None, # 3149 + None, # 3150 + None, # 3151 + None, # 3152 + None, # 3153 + None, # 3154 + None, # 3155 + None, # 3156 + None, # 3157 + None, # 3158 + None, # 3159 + None, # 3160 + None, # 3161 + None, # 3162 + None, # 3163 + None, # 3164 + None, # 3165 + None, # 3166 + None, # 3167 + None, # 3168 + None, # 3169 + None, # 3170 + None, # 3171 + None, # 3172 + None, # 3173 + None, # 3174 + None, # 3175 + None, # 3176 + None, # 3177 + None, # 3178 + None, # 3179 + None, # 3180 + None, # 3181 + None, # 3182 + None, # 3183 + None, # 3184 + None, # 3185 + None, # 3186 + None, # 3187 + None, # 3188 + None, # 3189 + None, # 3190 + None, # 3191 + None, # 3192 + None, # 3193 + None, # 3194 + None, # 3195 + None, # 3196 + None, # 3197 + None, # 3198 + None, # 3199 + None, # 3200 + None, # 3201 + None, # 3202 + None, # 3203 + None, # 3204 + None, # 3205 + None, # 3206 + None, # 3207 + None, # 3208 + None, # 3209 + None, # 3210 + None, # 3211 + None, # 3212 + None, # 3213 + None, # 3214 + None, # 3215 + None, # 3216 + None, # 3217 + None, # 3218 + None, # 3219 + None, # 3220 + None, # 3221 + None, # 3222 + None, # 3223 + None, # 3224 + None, # 3225 + None, # 3226 + None, # 3227 + None, # 3228 + None, # 3229 + None, # 3230 + None, # 3231 + None, # 3232 + None, # 3233 + None, # 3234 + None, # 3235 + None, # 3236 + None, # 3237 + None, # 3238 + None, # 3239 + None, # 3240 + None, # 3241 + None, # 3242 + None, # 3243 + None, # 3244 + None, # 3245 + None, # 3246 + None, # 3247 + None, # 3248 + None, # 3249 + None, # 3250 + None, # 3251 + None, # 3252 + None, # 3253 + None, # 3254 + None, # 3255 + None, # 3256 + None, # 3257 + None, # 3258 + None, # 3259 + None, # 3260 + None, # 3261 + None, # 3262 + None, # 3263 + None, # 3264 + None, # 3265 + None, # 3266 + None, # 3267 + None, # 3268 + None, # 3269 + None, # 3270 + None, # 3271 + None, # 3272 + None, # 3273 + None, # 3274 + None, # 3275 + None, # 3276 + None, # 3277 + None, # 3278 + None, # 3279 + None, # 3280 + None, # 3281 + None, # 3282 + None, # 3283 + None, # 3284 + None, # 3285 + None, # 3286 + None, # 3287 + None, # 3288 + None, # 3289 + None, # 3290 + None, # 3291 + None, # 3292 + None, # 3293 + None, # 3294 + None, # 3295 + None, # 3296 + None, # 3297 + None, # 3298 + None, # 3299 + None, # 3300 + None, # 3301 + None, # 3302 + None, # 3303 + None, # 3304 + None, # 3305 + None, # 3306 + None, # 3307 + None, # 3308 + None, # 3309 + None, # 3310 + None, # 3311 + None, # 3312 + None, # 3313 + None, # 3314 + None, # 3315 + None, # 3316 + None, # 3317 + None, # 3318 + None, # 3319 + None, # 3320 + None, # 3321 + None, # 3322 + None, # 3323 + None, # 3324 + None, # 3325 + None, # 3326 + None, # 3327 + None, # 3328 + (3329, TType.LIST, 'cloudFetchResults', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3329 ) all_structs.append(TDBSqlTempView) TDBSqlTempView.thrift_spec = ( @@ -12726,7 +15334,7 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - None, # 1281 + (1281, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1281 None, # 1282 None, # 1283 None, # 1284 @@ -29458,6 +32066,7 @@ def __ne__(self, other): (2, TType.BOOL, 'decimalAsArrow', None, None, ), # 2 (3, TType.BOOL, 'complexTypesAsArrow', None, None, ), # 3 (4, TType.BOOL, 'intervalTypesAsArrow', None, None, ), # 4 + (5, TType.BOOL, 'nullTypeAsArrow', None, None, ), # 5 ) all_structs.append(TExecuteStatementReq) TExecuteStatementReq.thrift_spec = ( @@ -30749,15 +33358,15 @@ def __ne__(self, other): (1285, TType.I64, 'maxBytesPerFile', None, None, ), # 1285 (1286, TType.STRUCT, 'useArrowNativeTypes', [TSparkArrowTypes, None], None, ), # 1286 (1287, TType.I64, 'resultRowLimit', None, None, ), # 1287 - None, # 1288 - None, # 1289 + (1288, TType.LIST, 'parameters', (TType.STRUCT, [TSparkParameter, None], False), None, ), # 1288 + (1289, TType.I64, 'maxBytesPerBatch', None, None, ), # 1289 None, # 1290 None, # 1291 None, # 1292 None, # 1293 None, # 1294 None, # 1295 - None, # 1296 + (1296, TType.STRUCT, 'statementConf', [TStatementConf, None], None, ), # 1296 None, # 1297 None, # 1298 None, # 1299 @@ -32798,6 +35407,51 @@ def __ne__(self, other): (3334, TType.STRING, 'requestValidation', 'BINARY', None, ), # 3334 (3335, TType.I32, 'resultPersistenceMode', None, None, ), # 3335 (3336, TType.BOOL, 'trimArrowBatchesToLimit', None, None, ), # 3336 + (3337, TType.I32, 'fetchDisposition', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.BOOL, 'enforceResultPersistenceMode', None, None, ), # 3344 + (3345, TType.LIST, 'statementList', (TType.STRUCT, [TDBSqlStatement, None], False), None, ), # 3345 + (3346, TType.BOOL, 'persistResultManifest', None, None, ), # 3346 + (3347, TType.I64, 'resultRetentionSeconds', None, None, ), # 3347 + (3348, TType.I64, 'resultByteLimit', None, None, ), # 3348 + (3349, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3349 + (3350, TType.STRING, 'originatingClientIdentity', 'UTF8', None, ), # 3350 + (3351, TType.BOOL, 'preferSingleFileResult', None, None, ), # 3351 + (3352, TType.BOOL, 'preferDriverOnlyUpload', None, None, ), # 3352 + (3353, TType.BOOL, 'enforceEmbeddedSchemaCorrectness', None, False, ), # 3353 +) +all_structs.append(TDBSqlStatement) +TDBSqlStatement.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'statement', 'UTF8', None, ), # 1 +) +all_structs.append(TSparkParameterValue) +TSparkParameterValue.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'stringValue', 'UTF8', None, ), # 1 + (2, TType.DOUBLE, 'doubleValue', None, None, ), # 2 + (3, TType.BOOL, 'booleanValue', None, None, ), # 3 +) +all_structs.append(TSparkParameter) +TSparkParameter.thrift_spec = ( + None, # 0 + (1, TType.I32, 'ordinal', None, None, ), # 1 + (2, TType.STRING, 'name', 'UTF8', None, ), # 2 + (3, TType.STRING, 'type', 'UTF8', None, ), # 3 + (4, TType.STRUCT, 'value', [TSparkParameterValue, None], None, ), # 4 +) +all_structs.append(TStatementConf) +TStatementConf.thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'sessionless', None, None, ), # 1 + (2, TType.STRUCT, 'initialNamespace', [TNamespace, None], None, ), # 2 + (3, TType.I32, 'client_protocol', None, None, ), # 3 + (4, TType.I64, 'client_protocol_i64', None, None, ), # 4 ) all_structs.append(TExecuteStatementResp) TExecuteStatementResp.thrift_spec = ( @@ -36136,6 +38790,9 @@ def __ne__(self, other): (3332, TType.STRUCT, 'sessionConf', [TDBSqlSessionConf, None], None, ), # 3332 (3333, TType.DOUBLE, 'currentClusterLoad', None, None, ), # 3333 (3334, TType.I32, 'idempotencyType', None, None, ), # 3334 + (3335, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3335 + (3336, TType.BOOL, 'isServerless', None, None, ), # 3336 + (3337, TType.LIST, 'operationHandles', (TType.STRUCT, [TOperationHandle, None], False), None, ), # 3337 ) all_structs.append(TGetTypeInfoReq) TGetTypeInfoReq.thrift_spec = ( @@ -78999,7 +81656,7 @@ def __ne__(self, other): None, # 1280 (1281, TType.STRING, 'displayMessage', 'UTF8', None, ), # 1281 (1282, TType.STRING, 'diagnosticInfo', 'UTF8', None, ), # 1282 - None, # 1283 + (1283, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1283 None, # 1284 None, # 1285 None, # 1286 @@ -91066,7 +93723,21 @@ def __ne__(self, other): (3329, TType.I32, 'reasonForNoCloudFetch', None, None, ), # 3329 (3330, TType.LIST, 'resultFiles', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3330 (3331, TType.STRING, 'manifestFile', 'UTF8', None, ), # 3331 - (3332, TType.STRING, 'manifestFileFormat', 'UTF8', None, ), # 3332 + (3332, TType.I32, 'manifestFileFormat', None, None, ), # 3332 + (3333, TType.I64, 'cacheLookupLatency', None, None, ), # 3333 + (3334, TType.STRING, 'remoteCacheMissReason', 'UTF8', None, ), # 3334 + (3335, TType.I32, 'fetchDisposition', None, None, ), # 3335 + (3336, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3336 + (3337, TType.BOOL, 'isServerless', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3344 + (3345, TType.BOOL, 'truncatedByThriftLimit', None, None, ), # 3345 + (3346, TType.I64, 'resultByteLimit', None, None, ), # 3346 ) all_structs.append(TFetchResultsReq) TFetchResultsReq.thrift_spec = ( @@ -105713,50 +108384,5 @@ def __ne__(self, other): (5, TType.STRING, 'footerSummary', 'UTF8', None, ), # 5 (6, TType.I64, 'startTime', None, None, ), # 6 ) -all_structs.append(TDBSqlClusterMetrics) -TDBSqlClusterMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'clusterCapacity', None, None, ), # 1 - (2, TType.I32, 'numRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numPendingTasks', None, None, ), # 3 - (4, TType.DOUBLE, 'rejectionThreshold', None, None, ), # 4 - (5, TType.DOUBLE, 'tasksCompletedPerMinute', None, None, ), # 5 -) -all_structs.append(TDBSqlQueryLaneMetrics) -TDBSqlQueryLaneMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'fastLaneReservation', None, None, ), # 1 - (2, TType.I32, 'numFastLaneRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numFastLanePendingTasks', None, None, ), # 3 - (4, TType.I32, 'slowLaneReservation', None, None, ), # 4 - (5, TType.I32, 'numSlowLaneRunningTasks', None, None, ), # 5 - (6, TType.I32, 'numSlowLanePendingTasks', None, None, ), # 6 -) -all_structs.append(TDBSqlQueryMetrics) -TDBSqlQueryMetrics.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 - (3, TType.I32, 'idempotencyType', None, None, ), # 3 - (4, TType.STRUCT, 'sessionHandle', [TSessionHandle, None], None, ), # 4 - (5, TType.I64, 'operationStarted', None, None, ), # 5 - (6, TType.DOUBLE, 'queryCost', None, None, ), # 6 - (7, TType.I32, 'numRunningTasks', None, None, ), # 7 - (8, TType.I32, 'numPendingTasks', None, None, ), # 8 - (9, TType.I32, 'numCompletedTasks', None, None, ), # 9 -) -all_structs.append(TDBSqlGetLoadInformationReq) -TDBSqlGetLoadInformationReq.thrift_spec = ( - None, # 0 - (1, TType.BOOL, 'includeQueryMetrics', None, False, ), # 1 -) -all_structs.append(TDBSqlGetLoadInformationResp) -TDBSqlGetLoadInformationResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'clusterMetrics', [TDBSqlClusterMetrics, None], None, ), # 2 - (3, TType.STRUCT, 'queryLaneMetrics', [TDBSqlQueryLaneMetrics, None], None, ), # 3 - (4, TType.LIST, 'queryMetrics', (TType.STRUCT, [TDBSqlQueryMetrics, None], False), None, ), # 4 -) fix_spec(all_structs) del all_structs From cac1da074b759e063bed6fbf68e54fa09215f9ff Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:55:10 -0700 Subject: [PATCH 2/7] Remove inject_parameters() Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- src/databricks/sql/client.py | 6 +--- src/databricks/sql/utils.py | 4 --- tests/unit/test_param_escaper.py | 49 +------------------------------- 3 files changed, 2 insertions(+), 57 deletions(-) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index eed13f1c5..6ca9951b6 100644 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -14,7 +14,7 @@ CursorAlreadyClosedError, ) from databricks.sql.thrift_backend import ThriftBackend -from databricks.sql.utils import ExecuteResponse, ParamEscaper, inject_parameters +from databricks.sql.utils import ExecuteResponse, ParamEscaper from databricks.sql.types import Row from databricks.sql.auth.auth import get_python_sql_connector_auth_provider from databricks.sql.experimental.oauth_persistence import OAuthPersistence @@ -493,10 +493,6 @@ def execute( Will result in the query "SELECT * FROM table WHERE field = 'foo' being sent to the server :returns self """ - if parameters is not None: - operation = inject_parameters( - operation, self.escaper.escape_args(parameters) - ) self._check_not_closed() self._close_and_clear_active_result_set() diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index 0aefc7a1c..854caa446 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -372,10 +372,6 @@ def escape_item(self, item): raise exc.ProgrammingError("Unsupported object {}".format(item)) -def inject_parameters(operation: str, parameters: Dict[str, str]): - return operation % parameters - - def create_arrow_table_from_arrow_file(file_bytes: bytes, description) -> pyarrow.Table: arrow_table = convert_arrow_based_file_to_arrow_table(file_bytes) return convert_decimals_in_arrow_table(arrow_table, description) diff --git a/tests/unit/test_param_escaper.py b/tests/unit/test_param_escaper.py index 6c1f1770e..ed10cc13f 100644 --- a/tests/unit/test_param_escaper.py +++ b/tests/unit/test_param_escaper.py @@ -1,7 +1,7 @@ from datetime import date, datetime import unittest, pytest, decimal -from databricks.sql.utils import ParamEscaper, inject_parameters +from databricks.sql.utils import ParamEscaper pe = ParamEscaper() @@ -106,50 +106,3 @@ def test_escape_sequence_sequence_of_strings(self): OUTPUT = "(('his','name'),('was','robert'),('palmer'))" assert pe.escape_sequence(INPUT) == OUTPUT - - -class TestFullQueryEscaping(object): - - def test_simple(self): - - INPUT = """ - SELECT - field1, - field2, - field3 - FROM - table - WHERE - field1 = %(param1)s - """ - - OUTPUT = """ - SELECT - field1, - field2, - field3 - FROM - table - WHERE - field1 = ';DROP ALL TABLES' - """ - - args = {"param1": ";DROP ALL TABLES"} - - assert inject_parameters(INPUT, pe.escape_args(args)) == OUTPUT - - @unittest.skipUnless(False, "Thrift server supports native parameter binding.") - def test_only_bind_in_where_clause(self): - - INPUT = """ - SELECT - %(field)s, - field2, - field3 - FROM table - """ - - args = {"field": "Some Value"} - - with pytest.raises(Exception): - inject_parameters(INPUT, pe.escape_args(args)) From d68f8cd8f8438c8600a7a6db66884f03aaf3cafc Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 14:58:24 -0700 Subject: [PATCH 3/7] ParamConverter to convert Dict to list of TSparkParameter Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- src/databricks/sql/client.py | 3 ++- src/databricks/sql/thrift_backend.py | 1 + src/databricks/sql/utils.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index 6ca9951b6..f584703b1 100644 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -482,7 +482,7 @@ def _handle_staging_remove(self, presigned_url: str, headers: dict = None): ) def execute( - self, operation: str, parameters: Optional[Dict[str, str]] = None + self, operation: str, parameters: Optional[Dict[str, Any]] = None ) -> "Cursor": """ Execute a query and wait for execution to complete. @@ -503,6 +503,7 @@ def execute( max_bytes=self.buffer_size_bytes, lz4_compression=self.connection.lz4_compression, cursor=self, + parameters=parameters, use_cloud_fetch=self.connection.use_cloud_fetch, ) self.active_result_set = ResultSet( diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 9f54aadbc..20335574c 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -811,6 +811,7 @@ def execute_command( max_bytes, lz4_compression, cursor, + parameters, use_cloud_fetch=False, ): assert session_handle is not None diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index 854caa446..ff8678522 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -15,6 +15,8 @@ TSparkArrowResultLink, TSparkRowSetType, TRowSet, + TSparkParameter, + TSparkParameterValue, ) BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] @@ -466,3 +468,30 @@ def _create_arrow_array(t_col_value_wrapper, arrow_type): result[i] = None return pyarrow.array(result, type=arrow_type) + +class ParamConverter: + + def convert_to_spark_parameters(self, parameters: Dict[str, Any]) -> List[TSparkParameter]: + spark_params = [] + for k,v in parameters.items(): + sql_type, spark_value = self._get_type_and_value(k, v) + spark_params.append( + TSparkParameter(name=k, type=sql_type, value=spark_value) + ) + return spark_params + + def _get_type_and_value(self, key: str, value: Any) -> (str, TSparkParameterValue): + if value is None: + raise RuntimeError("Parameter value for key {} is None".format(key)) + elif isinstance(value, bool): + return "BOOLEAN", TSparkParameterValue(booleanValue=value) + elif isinstance(value, int): + return "INT", TSparkParameterValue(doubleValue=value) + elif isinstance(value, str): + return "STRING", TSparkParameterValue(stringValue=value) + elif isinstance(value, float): + return "FLOAT", TSparkParameterValue(doubleValue=value) + # TODO: handle decimal.Decimal + # TODO: handle datetime.datetime + else: + raise exc.ProgrammingError("Unsupported parameter type for {}".format(value)) From 6d11a1df18fc2127b5d78bc63af3564870d88e7f Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:18:25 -0700 Subject: [PATCH 4/7] Change ParamConverter methods to static, add unit tests Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- src/databricks/sql/utils.py | 15 ++++++++----- tests/unit/test_param_converter.py | 35 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_param_converter.py diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index ff8678522..c19489862 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -469,20 +469,23 @@ def _create_arrow_array(t_col_value_wrapper, arrow_type): return pyarrow.array(result, type=arrow_type) + class ParamConverter: - def convert_to_spark_parameters(self, parameters: Dict[str, Any]) -> List[TSparkParameter]: + @staticmethod + def convert_to_spark_parameters(parameters: Dict[str, Any]) -> List[TSparkParameter]: spark_params = [] - for k,v in parameters.items(): - sql_type, spark_value = self._get_type_and_value(k, v) + for k, v in parameters.items(): + sql_type, spark_value = ParamConverter.get_type_and_value(v) spark_params.append( TSparkParameter(name=k, type=sql_type, value=spark_value) ) return spark_params - def _get_type_and_value(self, key: str, value: Any) -> (str, TSparkParameterValue): + @staticmethod + def get_type_and_value(value: Any) -> (str, TSparkParameterValue): if value is None: - raise RuntimeError("Parameter value for key {} is None".format(key)) + return "VOID", TSparkParameterValue() elif isinstance(value, bool): return "BOOLEAN", TSparkParameterValue(booleanValue=value) elif isinstance(value, int): @@ -490,7 +493,7 @@ def _get_type_and_value(self, key: str, value: Any) -> (str, TSparkParameterValu elif isinstance(value, str): return "STRING", TSparkParameterValue(stringValue=value) elif isinstance(value, float): - return "FLOAT", TSparkParameterValue(doubleValue=value) + return "DOUBLE", TSparkParameterValue(doubleValue=value) # TODO: handle decimal.Decimal # TODO: handle datetime.datetime else: diff --git a/tests/unit/test_param_converter.py b/tests/unit/test_param_converter.py new file mode 100644 index 000000000..2d6e7d2a1 --- /dev/null +++ b/tests/unit/test_param_converter.py @@ -0,0 +1,35 @@ +import unittest + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkParameter, TSparkParameterValue +from databricks.sql.utils import ParamConverter + + +class TestIndividualParameterTypes(unittest.TestCase): + def test_null_value(self): + assert ParamConverter.convert_to_spark_parameters({"key": None}) == \ + [TSparkParameter(name="key", type="VOID", value=TSparkParameterValue())] + + def test_string_value(self): + assert ParamConverter.convert_to_spark_parameters({"key": "value"}) == \ + [TSparkParameter(name="key", type="STRING", value=TSparkParameterValue(stringValue="value"))] + + def test_int_value(self): + assert ParamConverter.convert_to_spark_parameters({"key": 1}) == \ + [TSparkParameter(name="key", type="INT", value=TSparkParameterValue(doubleValue=1))] + + def test_float_value(self): + assert ParamConverter.convert_to_spark_parameters({"key": 1.0}) == \ + [TSparkParameter(name="key", type="DOUBLE", value=TSparkParameterValue(doubleValue=1.0))] + + def test_boolean_value(self): + assert ParamConverter.convert_to_spark_parameters({"key": True}) == \ + [TSparkParameter(name="key", type="BOOLEAN", value=TSparkParameterValue(booleanValue=True))] + + def test_multiple_named_parameters(self): + assert ParamConverter.convert_to_spark_parameters({"key_0": "value", "key_1": 1, "key_2": 1.0, "key_3": True}) \ + == [ + TSparkParameter(name="key_0", type="STRING", value=TSparkParameterValue(stringValue="value")), + TSparkParameter(name="key_1", type="INT", value=TSparkParameterValue(doubleValue=1)), + TSparkParameter(name="key_2", type="DOUBLE", value=TSparkParameterValue(doubleValue=1.0)), + TSparkParameter(name="key_3", type="BOOLEAN", value=TSparkParameterValue(booleanValue=True)) + ] From 8d3b3e82c467c4f75713b2d72cd083112dd1e7fd Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:27:30 -0700 Subject: [PATCH 5/7] Hookup ParamConverter to execute_statement Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- src/databricks/sql/client.py | 8 ++++---- src/databricks/sql/thrift_backend.py | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index f584703b1..46d04abbc 100644 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -486,11 +486,11 @@ def execute( ) -> "Cursor": """ Execute a query and wait for execution to complete. - Parameters should be given in extended param format style: %(...). + Named parameters should be given in this format: :param. For example: - operation = "SELECT * FROM table WHERE field = %(some_value)s" - parameters = {"some_value": "foo"} - Will result in the query "SELECT * FROM table WHERE field = 'foo' being sent to the server + operation = "SELECT * FROM table WHERE field = :some_param" + parameters = {"some_param": "foo"} + Will result in the query with the parameters sent to the server for server-side parameterization :returns self """ diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 20335574c..548848cff 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -35,6 +35,7 @@ convert_arrow_based_set_to_arrow_table, convert_decimals_in_arrow_table, convert_column_based_set_to_arrow_table, + ParamConverter ) logger = logging.getLogger(__name__) @@ -824,6 +825,7 @@ def execute_command( # DBR should be changed to use month_day_nano_interval intervalTypesAsArrow=False, ) + spark_parameters = ParamConverter.convert_to_spark_parameters(parameters) if parameters else None req = ttypes.TExecuteStatementReq( sessionHandle=session_handle, statement=operation, @@ -839,6 +841,7 @@ def execute_command( "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false" }, useArrowNativeTypes=spark_arrow_types, + parameters=spark_parameters, ) resp = self.make_request(self._client.ExecuteStatement, req) return self._handle_execute_response(resp, cursor) From 19fe0db40cf0a690738dfd6cf003e16e328e5ae9 Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:36:39 -0700 Subject: [PATCH 6/7] Ensure current unit tests succeed Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- tests/unit/test_thrift_backend.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index 3668213c1..743d6f764 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -400,7 +400,7 @@ def test_get_status_uses_display_message_if_available(self, tcli_service_class): thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) with self.assertRaises(DatabaseError) as cm: - thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) + thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock(), None) self.assertEqual(display_message, str(cm.exception)) self.assertIn(diagnostic_info, str(cm.exception.message_with_context())) @@ -430,7 +430,7 @@ def test_direct_results_uses_display_message_if_available(self, tcli_service_cla thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) with self.assertRaises(DatabaseError) as cm: - thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) + thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock(), None) self.assertEqual(display_message, str(cm.exception)) self.assertIn(diagnostic_info, str(cm.exception.message_with_context())) @@ -718,7 +718,7 @@ def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_s thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() - thrift_backend.execute_command("foo", Mock(), 100, 200, Mock(), cursor_mock) + thrift_backend.execute_command("foo", Mock(), 100, 200, Mock(), cursor_mock, None) # Check call to client req = tcli_service_instance.ExecuteStatement.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -877,7 +877,7 @@ def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_cl thrift_backend = self._make_fake_thrift_backend() with self.assertRaises(OperationalError) as cm: - thrift_backend.execute_command("foo", Mock(), 100, 100, Mock(), Mock()) + thrift_backend.execute_command("foo", Mock(), 100, 100, Mock(), Mock(), None) self.assertIn("Expected results to be in Arrow or column based format", str(cm.exception)) def test_create_arrow_table_raises_error_for_unsupported_type(self): @@ -1447,7 +1447,7 @@ def test_execute_command_sets_complex_type_fields_correctly(self, mock_handle_ex complex_arg_types["_use_arrow_native_decimals"] = decimals thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **complex_arg_types) - thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) + thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock(), None) t_execute_statement_req = tcli_service_instance.ExecuteStatement.call_args[0][0] # If the value is unset, the native type should default to True self.assertEqual(t_execute_statement_req.useArrowNativeTypes.timestampAsArrow, From 4a1c59dd3b76d2787d37e7f703b6588dcdc04f18 Mon Sep 17 00:00:00 2001 From: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:53:46 -0700 Subject: [PATCH 7/7] Add test for execute_command with parameters Signed-off-by: Matthew Kim <11141331+mattdeekay@users.noreply.github.com> --- tests/unit/test_thrift_backend.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index 743d6f764..30006f518 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -727,6 +727,26 @@ def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_s # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) + def test_execute_statement_calls_client_and_handle_execute_response_with_parameters(self, tcli_service_class): + tcli_service_instance = tcli_service_class.return_value + response = Mock() + tcli_service_instance.ExecuteStatement.return_value = response + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend._handle_execute_response = Mock() + cursor_mock = Mock() + + thrift_backend.execute_command("foo", Mock(), 100, 200, Mock(), cursor_mock, {"param_name": "param_value"}) + # Check call to client + req = tcli_service_instance.ExecuteStatement.call_args[0][0] + get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) + self.assertEqual(req.getDirectResults, get_direct_results) + spark_parameters = [ttypes.TSparkParameter(name="param_name", type="STRING", value=ttypes.TSparkParameterValue(stringValue="param_value"))] + self.assertEqual(req.parameters, spark_parameters) + self.assertEqual(req.statement, "foo") + # Check response handling + thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_catalogs_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value