From 00fc3a04616c233d9616030bb93323da1a56100e Mon Sep 17 00:00:00 2001 From: Qin Xuye Date: Sat, 23 Feb 2019 08:27:24 +0800 Subject: [PATCH] Add TileDB support as external storage (#227) * add tiledb relative expressions * support execute fromtiledb * support executeo totiledb --- .travis.yml | 2 +- mars/lib/sparse/__init__.py | 6 +- mars/lib/sparse/matrix.py | 3 + mars/lib/sparse/vector.py | 24 +- mars/opcodes.py | 2 + mars/operands/datastore.py | 4 +- mars/serialize/protos/operand.proto | 3 + mars/serialize/protos/operand_pb2.py | 596 +++++++++--------- mars/tensor/__init__.py | 3 +- mars/tensor/core.py | 10 + mars/tensor/execution/core.py | 2 + mars/tensor/execution/datasource.py | 49 ++ mars/tensor/execution/datastore.py | 51 +- .../tests/test_datasource_execute.py | 81 ++- .../execution/tests/test_datastore_execute.py | 70 ++ mars/tensor/execution/utils.py | 35 + .../tensor/expressions/datasource/__init__.py | 1 + .../expressions/datasource/fromtiledb.py | 110 ++++ mars/tensor/expressions/datastore/__init__.py | 4 +- mars/tensor/expressions/datastore/core.py | 50 ++ mars/tensor/expressions/datastore/totiledb.py | 114 ++++ mars/tensor/expressions/datastore/utils.py | 49 ++ mars/tensor/expressions/tests/test_core.py | 2 +- .../expressions/tests/test_datasource.py | 106 ++++ .../expressions/tests/test_datastore.py | 103 +++ 25 files changed, 1172 insertions(+), 308 deletions(-) create mode 100644 mars/tensor/execution/tests/test_datastore_execute.py create mode 100644 mars/tensor/execution/utils.py create mode 100644 mars/tensor/expressions/datasource/fromtiledb.py create mode 100644 mars/tensor/expressions/datastore/core.py create mode 100644 mars/tensor/expressions/datastore/totiledb.py create mode 100644 mars/tensor/expressions/datastore/utils.py create mode 100644 mars/tensor/expressions/tests/test_datasource.py create mode 100644 mars/tensor/expressions/tests/test_datastore.py diff --git a/.travis.yml b/.travis.yml index 5a7b734c81..ae729aa3f5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,7 +55,7 @@ install: fi - pip install -r requirements-dev.txt && pip install -r requirements-extra.txt && - pip install virtualenv coveralls flake8 etcd-gevent + pip install virtualenv coveralls flake8 etcd-gevent tiledb - virtualenv testenv && source testenv/bin/activate - pip install -r requirements.txt && pip install pytest pytest-timeout - if [ -z "$DEFAULT_VENV" ]; then deactivate; else source $DEFAULT_VENV/bin/activate; fi diff --git a/mars/lib/sparse/__init__.py b/mars/lib/sparse/__init__.py index 919d646789..f5a333abd6 100644 --- a/mars/lib/sparse/__init__.py +++ b/mars/lib/sparse/__init__.py @@ -26,11 +26,11 @@ from .coo import COONDArray -def asarray(x): +def asarray(x, shape=None): from .core import issparse if issparse(x): - return SparseNDArray(x) + return SparseNDArray(x, shape=shape) return x @@ -460,7 +460,7 @@ def matmul(a, b, sparse=True, **_): def concatenate(tensors, axis=0): has_sparse = any(issparse(t) for t in tensors) if has_sparse: - tensors = [asarray(get_sparse_module(t).csr_matrix(t)) for t in tensors] + tensors = [asarray(get_sparse_module(t).csr_matrix(t), t.shape) for t in tensors] return reduce(lambda a, b: _call_bin('concatenate', a, b, axis=axis), tensors) diff --git a/mars/lib/sparse/matrix.py b/mars/lib/sparse/matrix.py index e6d61688be..cd7318d461 100644 --- a/mars/lib/sparse/matrix.py +++ b/mars/lib/sparse/matrix.py @@ -106,6 +106,9 @@ def __init__(self, spmatrix, shape=()): def ndim(self): return self.spmatrix.ndim + def tocsr(self): + return self + def toarray(self): return self.spmatrix.toarray() diff --git a/mars/lib/sparse/vector.py b/mars/lib/sparse/vector.py index a629f5b2e3..e594d45bb0 100644 --- a/mars/lib/sparse/vector.py +++ b/mars/lib/sparse/vector.py @@ -15,7 +15,7 @@ # limitations under the License. from .array import SparseNDArray -from .core import get_array_module, cp, cps, naked, issparse +from .core import get_array_module, get_sparse_module, cp, cps, naked, issparse class SparseVector(SparseNDArray): @@ -39,6 +39,9 @@ def toarray(self): def todense(self): return self.spmatrix.toarray().reshape(self.shape) + def tocsr(self): + return self + def ascupy(self): is_cp = get_array_module(self.spmatrix) is cp if is_cp: @@ -118,3 +121,22 @@ def dot(self, other, sparse=True): shape = (other.shape[1],) return SparseNDArray(x, shape=shape) return get_array_module(x).asarray(x) + + def concatenate(self, other, axis=0): + try: + other = naked(other) + except TypeError: + return NotImplemented + + if issparse(other): + xps = get_sparse_module(self.spmatrix) + if axis != 0: + raise ValueError('axis can only be 0') + x = xps.hstack((self.spmatrix, other)) + else: + xp = get_array_module(self.spmatrix) + x = xp.concatenate((self.spmatrix.toarray().reshape(self.shape), other), axis=axis) + + if issparse(x): + return SparseNDArray(x, shape=(x.shape[1],)) + return get_array_module(x).asarray(x) diff --git a/mars/opcodes.py b/mars/opcodes.py index 95fecf7fc7..e608eaac90 100644 --- a/mars/opcodes.py +++ b/mars/opcodes.py @@ -16,6 +16,8 @@ TENSOR_LINSPACE = 14 TENSOR_TRIU = 15 TENSOR_TRIL = 16 +TENSOR_FROM_TILEDB = 18 +TENSOR_STORE_TILEDB = 19 DATAFRAME_DATA_SOURCE = 17 RAND_RAND = 41 RAND_RANDN = 42 diff --git a/mars/operands/datastore.py b/mars/operands/datastore.py index f890a8127a..99cee60660 100644 --- a/mars/operands/datastore.py +++ b/mars/operands/datastore.py @@ -14,8 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .core import Operand +from .base import HasInput -class DataStore(Operand): +class DataStore(HasInput): __slots__ = () diff --git a/mars/serialize/protos/operand.proto b/mars/serialize/protos/operand.proto index 2fa575a283..d6db76488a 100644 --- a/mars/serialize/protos/operand.proto +++ b/mars/serialize/protos/operand.proto @@ -26,6 +26,9 @@ message OperandDef { TENSOR_LINSPACE = 14; TENSOR_TRIU = 15; TENSOR_TRIL = 16; + // external storage + TENSOR_FROM_TILEDB = 18; + TENSOR_STORE_TILEDB = 19; // dataframe DATAFRAME_DATA_SOURCE = 17; diff --git a/mars/serialize/protos/operand_pb2.py b/mars/serialize/protos/operand_pb2.py index 01ae6274e0..df3c89ecba 100644 --- a/mars/serialize/protos/operand_pb2.py +++ b/mars/serialize/protos/operand_pb2.py @@ -20,7 +20,7 @@ package='', syntax='proto3', serialized_options=None, - serialized_pb=_b('\n#mars/serialize/protos/operand.proto\x1a!mars/serialize/protos/value.proto\"\x9e\'\n\nOperandDef\x12\x0c\n\x04type\x18\x01 \x01(\t\x12#\n\x04\x61ttr\x18\x02 \x03(\x0b\x32\x15.OperandDef.AttrEntry\x12\x1d\n\x08operands\x18\x03 \x03(\x0b\x32\x0b.OperandDef\x12\n\n\x02id\x18\x04 \x01(\t\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\"\xfc%\n\x0bOperandType\x12\x08\n\x04NULL\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x16\n\x12TENSOR_DATA_SOURCE\x10\x02\x12\x0f\n\x0bTENSOR_ONES\x10\x03\x12\x14\n\x10TENSOR_ONES_LIKE\x10\x04\x12\x10\n\x0cTENSOR_ZEROS\x10\x05\x12\x15\n\x11TENSOR_ZEROS_LIKE\x10\x06\x12\x10\n\x0cTENSOR_EMPTY\x10\x07\x12\x15\n\x11TENSOR_EMPTY_LIKE\x10\x08\x12\x0f\n\x0bTENSOR_FULL\x10\t\x12\x11\n\rTENSOR_ARANGE\x10\n\x12\x12\n\x0eTENSOR_INDICES\x10\x0b\x12\x0f\n\x0bTENSOR_DIAG\x10\x0c\x12\x0e\n\nTENSOR_EYE\x10\r\x12\x13\n\x0fTENSOR_LINSPACE\x10\x0e\x12\x0f\n\x0bTENSOR_TRIU\x10\x0f\x12\x0f\n\x0bTENSOR_TRIL\x10\x10\x12\x19\n\x15\x44\x41TAFRAME_DATA_SOURCE\x10\x11\x12\r\n\tRAND_RAND\x10)\x12\x0e\n\nRAND_RANDN\x10*\x12\x10\n\x0cRAND_RANDINT\x10+\x12\x18\n\x14RAND_RANDOM_INTEGERS\x10,\x12\x16\n\x12RAND_RANDOM_SAMPLE\x10-\x12\x0f\n\x0bRAND_RANDOM\x10.\x12\r\n\tRAND_RANF\x10/\x12\x0f\n\x0bRAND_SAMPLE\x10\x30\x12\x0e\n\nRAND_BYTES\x10\x31\x12\r\n\tRAND_BETA\x10\x32\x12\x11\n\rRAND_BINOMIAL\x10\x33\x12\x12\n\x0eRAND_CHISQUARE\x10\x34\x12\x0f\n\x0bRAND_CHOICE\x10\x35\x12\x12\n\x0eRAND_DIRICHLET\x10\x36\x12\x14\n\x10RAND_EXPONENTIAL\x10\x37\x12\n\n\x06RAND_F\x10\x38\x12\x0e\n\nRAND_GAMMA\x10\x39\x12\x12\n\x0eRAND_GEOMETRIC\x10:\x12\x0f\n\x0bRAND_GUMBEL\x10;\x12\x17\n\x13RAND_HYPERGEOMETRIC\x10<\x12\x10\n\x0cRAND_LAPLACE\x10=\x12\x11\n\rRAND_LOGISTIC\x10>\x12\x12\n\x0eRAND_LOGNORMAL\x10?\x12\x12\n\x0eRAND_LOGSERIES\x10@\x12\x14\n\x10RAND_MULTINOMIAL\x10\x41\x12\x1c\n\x18RAND_MULTIVARIATE_NORMAL\x10\x42\x12\x1a\n\x16RAND_NEGATIVE_BINOMIAL\x10\x43\x12\x1c\n\x18RAND_NONCENTRAL_CHISQURE\x10\x44\x12\x15\n\x11RAND_NONCENTRAL_F\x10\x45\x12\x0f\n\x0bRAND_NORMAL\x10\x46\x12\x0f\n\x0bRAND_PARETO\x10G\x12\x14\n\x10RAND_PERMUTATION\x10H\x12\x10\n\x0cRAND_POSSION\x10I\x12\x0e\n\nRAND_POWER\x10J\x12\x11\n\rRAND_RAYLEIGH\x10K\x12\x10\n\x0cRAND_SHUFFLE\x10L\x12\x18\n\x14RAND_STANDARD_CAUCHY\x10M\x12\x1d\n\x19RAND_STANDARD_EXPONENTIAL\x10N\x12\x18\n\x14RAND_STANDARD_GAMMMA\x10O\x12\x18\n\x14RAND_STANDARD_NORMAL\x10P\x12\x13\n\x0fRAND_STANDARD_T\x10Q\x12\x11\n\rRAND_TOMAXINT\x10R\x12\x13\n\x0fRAND_TRIANGULAR\x10S\x12\x10\n\x0cRAND_UNIFORM\x10T\x12\x11\n\rRAND_VONMISES\x10U\x12\r\n\tRAND_WALD\x10V\x12\x10\n\x0cRAND_WEIBULL\x10W\x12\r\n\tRAND_ZIPF\x10X\x12\x07\n\x03\x41\x44\x44\x10\x65\x12\x10\n\x0c\x41\x44\x44_CONSTANT\x10\x66\x12\x07\n\x03SUB\x10g\x12\x10\n\x0cSUB_CONSTANT\x10h\x12\x07\n\x03MUL\x10i\x12\x10\n\x0cMUL_CONSTANT\x10j\x12\x07\n\x03\x44IV\x10k\x12\x10\n\x0c\x44IV_CONSTANT\x10l\x12\x0b\n\x07TRUEDIV\x10m\x12\x11\n\rTDIV_CONSTANT\x10n\x12\x0c\n\x08\x46LOORDIV\x10o\x12\x11\n\rFDIV_CONSTANT\x10p\x12\x07\n\x03POW\x10q\x12\x10\n\x0cPOW_CONSTANT\x10r\x12\x07\n\x03MOD\x10s\x12\x10\n\x0cMOD_CONSTANT\x10t\x12\x08\n\x04\x46MOD\x10u\x12\x11\n\rFMOD_CONSTANT\x10v\x12\r\n\tLOGADDEXP\x10w\x12\x10\n\x0cLAE_CONSTANT\x10x\x12\x0e\n\nLOGADDEXP2\x10y\x12\x11\n\rLAE2_CONSTANT\x10z\x12\x0c\n\x08NEGATIVE\x10{\x12\x0c\n\x08POSITIVE\x10|\x12\x0c\n\x08\x41\x42SOLUTE\x10}\x12\x08\n\x04\x46\x41\x42S\x10~\x12\x07\n\x03\x41\x42S\x10\x7f\x12\t\n\x04RINT\x10\x80\x01\x12\t\n\x04SIGN\x10\x81\x01\x12\t\n\x04\x43ONJ\x10\x82\x01\x12\x08\n\x03\x45XP\x10\x83\x01\x12\t\n\x04\x45XP2\x10\x84\x01\x12\x08\n\x03LOG\x10\x85\x01\x12\t\n\x04LOG2\x10\x86\x01\x12\n\n\x05LOG10\x10\x87\x01\x12\n\n\x05\x45XPM1\x10\x88\x01\x12\n\n\x05LOG1P\x10\x89\x01\x12\t\n\x04SQRT\x10\x8a\x01\x12\x0b\n\x06SQUARE\x10\x8b\x01\x12\t\n\x04\x43\x42RT\x10\x8c\x01\x12\x0f\n\nRECIPROCAL\x10\x8d\x01\x12\x07\n\x02\x45Q\x10\x8e\x01\x12\x10\n\x0b\x45Q_CONSTANT\x10\x8f\x01\x12\x07\n\x02NE\x10\x90\x01\x12\x10\n\x0bNE_CONSTANT\x10\x91\x01\x12\x07\n\x02LT\x10\x92\x01\x12\x10\n\x0bLT_CONSTANT\x10\x93\x01\x12\x07\n\x02LE\x10\x94\x01\x12\x10\n\x0bLE_CONSTANT\x10\x95\x01\x12\x07\n\x02GT\x10\x96\x01\x12\x10\n\x0bGT_CONSTANT\x10\x97\x01\x12\x07\n\x02GE\x10\x98\x01\x12\x10\n\x0bGE_CONSTANT\x10\x99\x01\x12\x08\n\x03SIN\x10\x9a\x01\x12\x08\n\x03\x43OS\x10\x9b\x01\x12\x08\n\x03TAN\x10\x9c\x01\x12\x0b\n\x06\x41RCSIN\x10\x9d\x01\x12\x0b\n\x06\x41RCCOS\x10\x9e\x01\x12\x0b\n\x06\x41RCTAN\x10\x9f\x01\x12\x0c\n\x07\x41RCTAN2\x10\xa0\x01\x12\x13\n\x0e\x41RCT2_CONSTANT\x10\xa1\x01\x12\n\n\x05HYPOT\x10\xa2\x01\x12\x13\n\x0eHYPOT_CONSTANT\x10\xa3\x01\x12\t\n\x04SINH\x10\xa4\x01\x12\t\n\x04\x43OSH\x10\xa5\x01\x12\t\n\x04TANH\x10\xa6\x01\x12\x0c\n\x07\x41RCSINH\x10\xa7\x01\x12\x0c\n\x07\x41RCCOSH\x10\xa8\x01\x12\x0c\n\x07\x41RCTANH\x10\xa9\x01\x12\x0c\n\x07\x44\x45G2RAD\x10\xaa\x01\x12\x0c\n\x07RAD2DEG\x10\xab\x01\x12\x0b\n\x06\x42ITAND\x10\xac\x01\x12\x14\n\x0f\x42ITAND_CONSTANT\x10\xad\x01\x12\n\n\x05\x42ITOR\x10\xae\x01\x12\x13\n\x0e\x42ITOR_CONSTANT\x10\xaf\x01\x12\x0b\n\x06\x42ITXOR\x10\xb0\x01\x12\x14\n\x0f\x42ITXOR_CONSTANT\x10\xb1\x01\x12\x0b\n\x06INVERT\x10\xb2\x01\x12\x0b\n\x06LSHIFT\x10\xb3\x01\x12\x14\n\x0fLSHIFT_CONSTANT\x10\xb4\x01\x12\x0b\n\x06RSHIFT\x10\xb5\x01\x12\x14\n\x0fRSHIFT_CONSTANT\x10\xb6\x01\x12\x08\n\x03\x41ND\x10\xb7\x01\x12\x11\n\x0c\x41ND_CONSTANT\x10\xb8\x01\x12\x07\n\x02OR\x10\xb9\x01\x12\x10\n\x0bOR_CONSTANT\x10\xba\x01\x12\x08\n\x03XOR\x10\xbb\x01\x12\x11\n\x0cXOR_CONSTANT\x10\xbc\x01\x12\x08\n\x03NOT\x10\xbd\x01\x12\x0c\n\x07MAXIMUM\x10\xbe\x01\x12\x15\n\x10MAXIMUM_CONSTANT\x10\xbf\x01\x12\x0c\n\x07MINIMUM\x10\xc0\x01\x12\x15\n\x10MINIMUM_CONSTANT\x10\xc1\x01\x12\x0b\n\x06\x41ROUND\x10\xc2\x01\x12\x10\n\x0b\x46LOAT_POWER\x10\xc3\x01\x12\x19\n\x14\x46LOAT_POWER_CONSTANT\x10\xc4\x01\x12\t\n\x04\x46MAX\x10\xc5\x01\x12\x12\n\rFMAX_CONSTANT\x10\xc6\x01\x12\t\n\x04\x46MIN\x10\xc7\x01\x12\x12\n\rFMIN_CONSTANT\x10\xc8\x01\x12\r\n\x08ISFINITE\x10\xc9\x01\x12\n\n\x05ISINF\x10\xca\x01\x12\n\n\x05ISNAN\x10\xcb\x01\x12\x0c\n\x07SIGNBIT\x10\xcc\x01\x12\r\n\x08\x43OPYSIGN\x10\xcd\x01\x12\x16\n\x11\x43OPYSIGN_CONSTANT\x10\xce\x01\x12\x0e\n\tNEXTAFTER\x10\xcf\x01\x12\x17\n\x12NEXTAFTER_CONSTANT\x10\xd0\x01\x12\x0c\n\x07SPACING\x10\xd1\x01\x12\n\n\x05LDEXP\x10\xd2\x01\x12\x13\n\x0eLDEXP_CONSTANT\x10\xd3\x01\x12\n\n\x05\x46REXP\x10\xd4\x01\x12\t\n\x04MODF\x10\xd5\x01\x12\n\n\x05\x46LOOR\x10\xd6\x01\x12\t\n\x04\x43\x45IL\x10\xd7\x01\x12\n\n\x05TRUNC\x10\xd8\x01\x12\x0c\n\x07\x44\x45GREES\x10\xd9\x01\x12\x0c\n\x07RADIANS\x10\xda\x01\x12\t\n\x04\x43LIP\x10\xdb\x01\x12\x0b\n\x06ISREAL\x10\xdc\x01\x12\x0e\n\tISCOMPLEX\x10\xdd\x01\x12\t\n\x04REAL\x10\xde\x01\x12\t\n\x04IMAG\x10\xdf\x01\x12\x08\n\x03\x46IX\x10\xe0\x01\x12\x07\n\x02I0\x10\xe1\x01\x12\t\n\x04SINC\x10\xe2\x01\x12\x0f\n\nNAN_TO_NUM\x10\xe3\x01\x12\x0c\n\x07ISCLOSE\x10\xe4\x01\x12\x15\n\x10ISCLOSE_CONSTANT\x10\xe5\x01\x12\x0b\n\x06\x44IVMOD\x10\xe7\x01\x12\n\n\x05\x41NGLE\x10\xe8\x01\x12\r\n\x08SET_REAL\x10\xe9\x01\x12\x16\n\x11SET_REAL_CONSTANT\x10\xea\x01\x12\r\n\x08SET_IMAG\x10\xeb\x01\x12\x16\n\x11SET_IMAG_CONSTANT\x10\xec\x01\x12\r\n\x08TREE_ADD\x10\xfb\x01\x12\x12\n\rTREE_MULTIPLY\x10\xfc\x01\x12\x0b\n\x06\x43UMSUM\x10\xad\x02\x12\x0c\n\x07\x43UMPROD\x10\xae\x02\x12\t\n\x04PROD\x10\xaf\x02\x12\x08\n\x03SUM\x10\xb0\x02\x12\x08\n\x03MAX\x10\xb1\x02\x12\x08\n\x03MIN\x10\xb2\x02\x12\x08\n\x03\x41LL\x10\xb3\x02\x12\x08\n\x03\x41NY\x10\xb4\x02\x12\x0f\n\nMEAN_CHUNK\x10\xb5\x02\x12\x11\n\x0cMEAN_COMBINE\x10\xb6\x02\x12\t\n\x04MEAN\x10\xb7\x02\x12\x0b\n\x06\x41RGMAX\x10\xb8\x02\x12\x11\n\x0c\x41RGMAX_CHUNK\x10\xb9\x02\x12\x13\n\x0e\x41RGMAX_COMBINE\x10\xba\x02\x12\x0b\n\x06\x41RGMIN\x10\xbb\x02\x12\x11\n\x0c\x41RGMIN_CHUNK\x10\xbc\x02\x12\x13\n\x0e\x41RGMIN_COMBINE\x10\xbd\x02\x12\x0b\n\x06NANSUM\x10\xbe\x02\x12\x0b\n\x06NANMAX\x10\xbf\x02\x12\x0b\n\x06NANMIN\x10\xc0\x02\x12\x0c\n\x07NANPROD\x10\xc1\x02\x12\x0c\n\x07NANMEAN\x10\xc2\x02\x12\x12\n\rNANMEAN_CHUNK\x10\xc3\x02\x12\x0e\n\tNANARGMAX\x10\xc4\x02\x12\x14\n\x0fNANARGMAX_CHUNK\x10\xc5\x02\x12\x16\n\x11NANARGMAX_COMBINE\x10\xc6\x02\x12\x0e\n\tNANARGMIN\x10\xc7\x02\x12\x14\n\x0fNANARGMIN_CHUNK\x10\xc8\x02\x12\x16\n\x11NANARGMIN_COMBINE\x10\xc9\x02\x12\x12\n\rCOUNT_NONZERO\x10\xca\x02\x12\x11\n\x0cMOMENT_CHUNK\x10\xcb\x02\x12\x14\n\x0fNANMOMENT_CHUNK\x10\xcc\x02\x12\x13\n\x0eMOMENT_COMBINE\x10\xcd\x02\x12\x16\n\x11NANMOMENT_COMBINE\x10\xce\x02\x12\x0b\n\x06MOMENT\x10\xcf\x02\x12\x0e\n\tNANMOMENT\x10\xd0\x02\x12\x08\n\x03VAR\x10\xd1\x02\x12\x08\n\x03STD\x10\xd2\x02\x12\x0b\n\x06NANVAR\x10\xd3\x02\x12\x0b\n\x06NANSTD\x10\xd4\x02\x12\x0e\n\tNANCUMSUM\x10\xd5\x02\x12\x0f\n\nNANCUMPROD\x10\xd6\x02\x12\x0c\n\x07RESHAPE\x10\x91\x03\x12\n\n\x05SLICE\x10\x92\x03\x12\n\n\x05INDEX\x10\x93\x03\x12\x12\n\rINDEXSETVALUE\x10\x94\x03\x12\x10\n\x0b\x43ONCATENATE\x10\x95\x03\x12\x0c\n\x07RECHUNK\x10\x96\x03\x12\x0b\n\x06\x41STYPE\x10\x97\x03\x12\x0e\n\tTRANSPOSE\x10\x98\x03\x12\r\n\x08SWAPAXES\x10\x99\x03\x12\x11\n\x0c\x42ROADCAST_TO\x10\x9a\x03\x12\n\n\x05STACK\x10\x9b\x03\x12\n\n\x05WHERE\x10\x9c\x03\x12\x0b\n\x06\x43HOOSE\x10\x9d\x03\x12\x0c\n\x07NONZERO\x10\x9e\x03\x12\r\n\x08\x41RGWHERE\x10\x9f\x03\x12\x12\n\rUNRAVEL_INDEX\x10\xa0\x03\x12\x16\n\x11RAVEL_MULTI_INDEX\x10\xa1\x03\x12\x10\n\x0b\x41RRAY_SPLIT\x10\xa2\x03\x12\x0c\n\x07SQUEEZE\x10\xa3\x03\x12\r\n\x08\x44IGITIZE\x10\xa4\x03\x12\x0b\n\x06REPEAT\x10\xa5\x03\x12\x0b\n\x06\x43OPYTO\x10\xa6\x03\x12\t\n\x04ISIN\x10\xa7\x03\x12\x0e\n\tTENSORDOT\x10\xf5\x03\x12\x08\n\x03\x44OT\x10\xf6\x03\x12\x0b\n\x06MATMUL\x10\xf7\x03\x12\r\n\x08\x43HOLESKY\x10\xfe\x03\x12\x07\n\x02QR\x10\xff\x03\x12\x08\n\x03SVD\x10\x80\x04\x12\x07\n\x02LU\x10\x81\x04\x12\x15\n\x10SOLVE_TRIANGULAR\x10\x88\x04\x12\t\n\x04NORM\x10\x92\x04\x12\x08\n\x03\x46\x46T\x10\xd9\x04\x12\t\n\x04IFFT\x10\xda\x04\x12\t\n\x04\x46\x46T2\x10\xdb\x04\x12\n\n\x05IFFT2\x10\xdc\x04\x12\t\n\x04\x46\x46TN\x10\xdd\x04\x12\n\n\x05IFFTN\x10\xde\x04\x12\t\n\x04RFFT\x10\xdf\x04\x12\n\n\x05IRFFT\x10\xe0\x04\x12\n\n\x05RFFT2\x10\xe1\x04\x12\x0b\n\x06IRFFT2\x10\xe2\x04\x12\n\n\x05RFFTN\x10\xe3\x04\x12\x0b\n\x06IRFFTN\x10\xe4\x04\x12\t\n\x04HFFT\x10\xe5\x04\x12\n\n\x05IHFFT\x10\xe6\x04\x12\x0c\n\x07\x46\x46TFREQ\x10\xe7\x04\x12\x12\n\rFFTFREQ_CHUNK\x10\xe8\x04\x12\r\n\x08RFFTFREQ\x10\xe9\x04\x12\r\n\x08\x46\x46TSHIFT\x10\xea\x04\x12\x0e\n\tIFFTSHIFT\x10\xeb\x04\x12\x1e\n\x19SPARSE_MATRIX_DATA_SOURCE\x10\xbd\x05\x12\x14\n\x0f\x44\x45NSE_TO_SPARSE\x10\xbe\x05\x12\x14\n\x0fSPARSE_TO_DENSE\x10\xbf\x05\x12\t\n\x04\x46USE\x10\xa1\x06\x12\n\n\x05\x45NTER\x10\x85\x07\x12\n\n\x05LEAVE\x10\x86\x07\x12\x0f\n\nFIX_LATEST\x10\x87\x07\x12\x0c\n\x07IF_ELSE\x10\x88\x07\x12\x0e\n\tNEXT_ITER\x10\x89\x07\x12\x0e\n\tTABLE_COO\x10\xeb\x07\x12\x0e\n\tSTORE_COO\x10\xec\x07\x12\x0b\n\x05\x46\x45TCH\x10\xbf\x84=b\x06proto3') + serialized_pb=_b('\n#mars/serialize/protos/operand.proto\x1a!mars/serialize/protos/value.proto\"\xcf\'\n\nOperandDef\x12\x0c\n\x04type\x18\x01 \x01(\t\x12#\n\x04\x61ttr\x18\x02 \x03(\x0b\x32\x15.OperandDef.AttrEntry\x12\x1d\n\x08operands\x18\x03 \x03(\x0b\x32\x0b.OperandDef\x12\n\n\x02id\x18\x04 \x01(\t\x1a\x33\n\tAttrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x15\n\x05value\x18\x02 \x01(\x0b\x32\x06.Value:\x02\x38\x01\"\xad&\n\x0bOperandType\x12\x08\n\x04NULL\x10\x00\x12\n\n\x06SCALAR\x10\x01\x12\x16\n\x12TENSOR_DATA_SOURCE\x10\x02\x12\x0f\n\x0bTENSOR_ONES\x10\x03\x12\x14\n\x10TENSOR_ONES_LIKE\x10\x04\x12\x10\n\x0cTENSOR_ZEROS\x10\x05\x12\x15\n\x11TENSOR_ZEROS_LIKE\x10\x06\x12\x10\n\x0cTENSOR_EMPTY\x10\x07\x12\x15\n\x11TENSOR_EMPTY_LIKE\x10\x08\x12\x0f\n\x0bTENSOR_FULL\x10\t\x12\x11\n\rTENSOR_ARANGE\x10\n\x12\x12\n\x0eTENSOR_INDICES\x10\x0b\x12\x0f\n\x0bTENSOR_DIAG\x10\x0c\x12\x0e\n\nTENSOR_EYE\x10\r\x12\x13\n\x0fTENSOR_LINSPACE\x10\x0e\x12\x0f\n\x0bTENSOR_TRIU\x10\x0f\x12\x0f\n\x0bTENSOR_TRIL\x10\x10\x12\x16\n\x12TENSOR_FROM_TILEDB\x10\x12\x12\x17\n\x13TENSOR_STORE_TILEDB\x10\x13\x12\x19\n\x15\x44\x41TAFRAME_DATA_SOURCE\x10\x11\x12\r\n\tRAND_RAND\x10)\x12\x0e\n\nRAND_RANDN\x10*\x12\x10\n\x0cRAND_RANDINT\x10+\x12\x18\n\x14RAND_RANDOM_INTEGERS\x10,\x12\x16\n\x12RAND_RANDOM_SAMPLE\x10-\x12\x0f\n\x0bRAND_RANDOM\x10.\x12\r\n\tRAND_RANF\x10/\x12\x0f\n\x0bRAND_SAMPLE\x10\x30\x12\x0e\n\nRAND_BYTES\x10\x31\x12\r\n\tRAND_BETA\x10\x32\x12\x11\n\rRAND_BINOMIAL\x10\x33\x12\x12\n\x0eRAND_CHISQUARE\x10\x34\x12\x0f\n\x0bRAND_CHOICE\x10\x35\x12\x12\n\x0eRAND_DIRICHLET\x10\x36\x12\x14\n\x10RAND_EXPONENTIAL\x10\x37\x12\n\n\x06RAND_F\x10\x38\x12\x0e\n\nRAND_GAMMA\x10\x39\x12\x12\n\x0eRAND_GEOMETRIC\x10:\x12\x0f\n\x0bRAND_GUMBEL\x10;\x12\x17\n\x13RAND_HYPERGEOMETRIC\x10<\x12\x10\n\x0cRAND_LAPLACE\x10=\x12\x11\n\rRAND_LOGISTIC\x10>\x12\x12\n\x0eRAND_LOGNORMAL\x10?\x12\x12\n\x0eRAND_LOGSERIES\x10@\x12\x14\n\x10RAND_MULTINOMIAL\x10\x41\x12\x1c\n\x18RAND_MULTIVARIATE_NORMAL\x10\x42\x12\x1a\n\x16RAND_NEGATIVE_BINOMIAL\x10\x43\x12\x1c\n\x18RAND_NONCENTRAL_CHISQURE\x10\x44\x12\x15\n\x11RAND_NONCENTRAL_F\x10\x45\x12\x0f\n\x0bRAND_NORMAL\x10\x46\x12\x0f\n\x0bRAND_PARETO\x10G\x12\x14\n\x10RAND_PERMUTATION\x10H\x12\x10\n\x0cRAND_POSSION\x10I\x12\x0e\n\nRAND_POWER\x10J\x12\x11\n\rRAND_RAYLEIGH\x10K\x12\x10\n\x0cRAND_SHUFFLE\x10L\x12\x18\n\x14RAND_STANDARD_CAUCHY\x10M\x12\x1d\n\x19RAND_STANDARD_EXPONENTIAL\x10N\x12\x18\n\x14RAND_STANDARD_GAMMMA\x10O\x12\x18\n\x14RAND_STANDARD_NORMAL\x10P\x12\x13\n\x0fRAND_STANDARD_T\x10Q\x12\x11\n\rRAND_TOMAXINT\x10R\x12\x13\n\x0fRAND_TRIANGULAR\x10S\x12\x10\n\x0cRAND_UNIFORM\x10T\x12\x11\n\rRAND_VONMISES\x10U\x12\r\n\tRAND_WALD\x10V\x12\x10\n\x0cRAND_WEIBULL\x10W\x12\r\n\tRAND_ZIPF\x10X\x12\x07\n\x03\x41\x44\x44\x10\x65\x12\x10\n\x0c\x41\x44\x44_CONSTANT\x10\x66\x12\x07\n\x03SUB\x10g\x12\x10\n\x0cSUB_CONSTANT\x10h\x12\x07\n\x03MUL\x10i\x12\x10\n\x0cMUL_CONSTANT\x10j\x12\x07\n\x03\x44IV\x10k\x12\x10\n\x0c\x44IV_CONSTANT\x10l\x12\x0b\n\x07TRUEDIV\x10m\x12\x11\n\rTDIV_CONSTANT\x10n\x12\x0c\n\x08\x46LOORDIV\x10o\x12\x11\n\rFDIV_CONSTANT\x10p\x12\x07\n\x03POW\x10q\x12\x10\n\x0cPOW_CONSTANT\x10r\x12\x07\n\x03MOD\x10s\x12\x10\n\x0cMOD_CONSTANT\x10t\x12\x08\n\x04\x46MOD\x10u\x12\x11\n\rFMOD_CONSTANT\x10v\x12\r\n\tLOGADDEXP\x10w\x12\x10\n\x0cLAE_CONSTANT\x10x\x12\x0e\n\nLOGADDEXP2\x10y\x12\x11\n\rLAE2_CONSTANT\x10z\x12\x0c\n\x08NEGATIVE\x10{\x12\x0c\n\x08POSITIVE\x10|\x12\x0c\n\x08\x41\x42SOLUTE\x10}\x12\x08\n\x04\x46\x41\x42S\x10~\x12\x07\n\x03\x41\x42S\x10\x7f\x12\t\n\x04RINT\x10\x80\x01\x12\t\n\x04SIGN\x10\x81\x01\x12\t\n\x04\x43ONJ\x10\x82\x01\x12\x08\n\x03\x45XP\x10\x83\x01\x12\t\n\x04\x45XP2\x10\x84\x01\x12\x08\n\x03LOG\x10\x85\x01\x12\t\n\x04LOG2\x10\x86\x01\x12\n\n\x05LOG10\x10\x87\x01\x12\n\n\x05\x45XPM1\x10\x88\x01\x12\n\n\x05LOG1P\x10\x89\x01\x12\t\n\x04SQRT\x10\x8a\x01\x12\x0b\n\x06SQUARE\x10\x8b\x01\x12\t\n\x04\x43\x42RT\x10\x8c\x01\x12\x0f\n\nRECIPROCAL\x10\x8d\x01\x12\x07\n\x02\x45Q\x10\x8e\x01\x12\x10\n\x0b\x45Q_CONSTANT\x10\x8f\x01\x12\x07\n\x02NE\x10\x90\x01\x12\x10\n\x0bNE_CONSTANT\x10\x91\x01\x12\x07\n\x02LT\x10\x92\x01\x12\x10\n\x0bLT_CONSTANT\x10\x93\x01\x12\x07\n\x02LE\x10\x94\x01\x12\x10\n\x0bLE_CONSTANT\x10\x95\x01\x12\x07\n\x02GT\x10\x96\x01\x12\x10\n\x0bGT_CONSTANT\x10\x97\x01\x12\x07\n\x02GE\x10\x98\x01\x12\x10\n\x0bGE_CONSTANT\x10\x99\x01\x12\x08\n\x03SIN\x10\x9a\x01\x12\x08\n\x03\x43OS\x10\x9b\x01\x12\x08\n\x03TAN\x10\x9c\x01\x12\x0b\n\x06\x41RCSIN\x10\x9d\x01\x12\x0b\n\x06\x41RCCOS\x10\x9e\x01\x12\x0b\n\x06\x41RCTAN\x10\x9f\x01\x12\x0c\n\x07\x41RCTAN2\x10\xa0\x01\x12\x13\n\x0e\x41RCT2_CONSTANT\x10\xa1\x01\x12\n\n\x05HYPOT\x10\xa2\x01\x12\x13\n\x0eHYPOT_CONSTANT\x10\xa3\x01\x12\t\n\x04SINH\x10\xa4\x01\x12\t\n\x04\x43OSH\x10\xa5\x01\x12\t\n\x04TANH\x10\xa6\x01\x12\x0c\n\x07\x41RCSINH\x10\xa7\x01\x12\x0c\n\x07\x41RCCOSH\x10\xa8\x01\x12\x0c\n\x07\x41RCTANH\x10\xa9\x01\x12\x0c\n\x07\x44\x45G2RAD\x10\xaa\x01\x12\x0c\n\x07RAD2DEG\x10\xab\x01\x12\x0b\n\x06\x42ITAND\x10\xac\x01\x12\x14\n\x0f\x42ITAND_CONSTANT\x10\xad\x01\x12\n\n\x05\x42ITOR\x10\xae\x01\x12\x13\n\x0e\x42ITOR_CONSTANT\x10\xaf\x01\x12\x0b\n\x06\x42ITXOR\x10\xb0\x01\x12\x14\n\x0f\x42ITXOR_CONSTANT\x10\xb1\x01\x12\x0b\n\x06INVERT\x10\xb2\x01\x12\x0b\n\x06LSHIFT\x10\xb3\x01\x12\x14\n\x0fLSHIFT_CONSTANT\x10\xb4\x01\x12\x0b\n\x06RSHIFT\x10\xb5\x01\x12\x14\n\x0fRSHIFT_CONSTANT\x10\xb6\x01\x12\x08\n\x03\x41ND\x10\xb7\x01\x12\x11\n\x0c\x41ND_CONSTANT\x10\xb8\x01\x12\x07\n\x02OR\x10\xb9\x01\x12\x10\n\x0bOR_CONSTANT\x10\xba\x01\x12\x08\n\x03XOR\x10\xbb\x01\x12\x11\n\x0cXOR_CONSTANT\x10\xbc\x01\x12\x08\n\x03NOT\x10\xbd\x01\x12\x0c\n\x07MAXIMUM\x10\xbe\x01\x12\x15\n\x10MAXIMUM_CONSTANT\x10\xbf\x01\x12\x0c\n\x07MINIMUM\x10\xc0\x01\x12\x15\n\x10MINIMUM_CONSTANT\x10\xc1\x01\x12\x0b\n\x06\x41ROUND\x10\xc2\x01\x12\x10\n\x0b\x46LOAT_POWER\x10\xc3\x01\x12\x19\n\x14\x46LOAT_POWER_CONSTANT\x10\xc4\x01\x12\t\n\x04\x46MAX\x10\xc5\x01\x12\x12\n\rFMAX_CONSTANT\x10\xc6\x01\x12\t\n\x04\x46MIN\x10\xc7\x01\x12\x12\n\rFMIN_CONSTANT\x10\xc8\x01\x12\r\n\x08ISFINITE\x10\xc9\x01\x12\n\n\x05ISINF\x10\xca\x01\x12\n\n\x05ISNAN\x10\xcb\x01\x12\x0c\n\x07SIGNBIT\x10\xcc\x01\x12\r\n\x08\x43OPYSIGN\x10\xcd\x01\x12\x16\n\x11\x43OPYSIGN_CONSTANT\x10\xce\x01\x12\x0e\n\tNEXTAFTER\x10\xcf\x01\x12\x17\n\x12NEXTAFTER_CONSTANT\x10\xd0\x01\x12\x0c\n\x07SPACING\x10\xd1\x01\x12\n\n\x05LDEXP\x10\xd2\x01\x12\x13\n\x0eLDEXP_CONSTANT\x10\xd3\x01\x12\n\n\x05\x46REXP\x10\xd4\x01\x12\t\n\x04MODF\x10\xd5\x01\x12\n\n\x05\x46LOOR\x10\xd6\x01\x12\t\n\x04\x43\x45IL\x10\xd7\x01\x12\n\n\x05TRUNC\x10\xd8\x01\x12\x0c\n\x07\x44\x45GREES\x10\xd9\x01\x12\x0c\n\x07RADIANS\x10\xda\x01\x12\t\n\x04\x43LIP\x10\xdb\x01\x12\x0b\n\x06ISREAL\x10\xdc\x01\x12\x0e\n\tISCOMPLEX\x10\xdd\x01\x12\t\n\x04REAL\x10\xde\x01\x12\t\n\x04IMAG\x10\xdf\x01\x12\x08\n\x03\x46IX\x10\xe0\x01\x12\x07\n\x02I0\x10\xe1\x01\x12\t\n\x04SINC\x10\xe2\x01\x12\x0f\n\nNAN_TO_NUM\x10\xe3\x01\x12\x0c\n\x07ISCLOSE\x10\xe4\x01\x12\x15\n\x10ISCLOSE_CONSTANT\x10\xe5\x01\x12\x0b\n\x06\x44IVMOD\x10\xe7\x01\x12\n\n\x05\x41NGLE\x10\xe8\x01\x12\r\n\x08SET_REAL\x10\xe9\x01\x12\x16\n\x11SET_REAL_CONSTANT\x10\xea\x01\x12\r\n\x08SET_IMAG\x10\xeb\x01\x12\x16\n\x11SET_IMAG_CONSTANT\x10\xec\x01\x12\r\n\x08TREE_ADD\x10\xfb\x01\x12\x12\n\rTREE_MULTIPLY\x10\xfc\x01\x12\x0b\n\x06\x43UMSUM\x10\xad\x02\x12\x0c\n\x07\x43UMPROD\x10\xae\x02\x12\t\n\x04PROD\x10\xaf\x02\x12\x08\n\x03SUM\x10\xb0\x02\x12\x08\n\x03MAX\x10\xb1\x02\x12\x08\n\x03MIN\x10\xb2\x02\x12\x08\n\x03\x41LL\x10\xb3\x02\x12\x08\n\x03\x41NY\x10\xb4\x02\x12\x0f\n\nMEAN_CHUNK\x10\xb5\x02\x12\x11\n\x0cMEAN_COMBINE\x10\xb6\x02\x12\t\n\x04MEAN\x10\xb7\x02\x12\x0b\n\x06\x41RGMAX\x10\xb8\x02\x12\x11\n\x0c\x41RGMAX_CHUNK\x10\xb9\x02\x12\x13\n\x0e\x41RGMAX_COMBINE\x10\xba\x02\x12\x0b\n\x06\x41RGMIN\x10\xbb\x02\x12\x11\n\x0c\x41RGMIN_CHUNK\x10\xbc\x02\x12\x13\n\x0e\x41RGMIN_COMBINE\x10\xbd\x02\x12\x0b\n\x06NANSUM\x10\xbe\x02\x12\x0b\n\x06NANMAX\x10\xbf\x02\x12\x0b\n\x06NANMIN\x10\xc0\x02\x12\x0c\n\x07NANPROD\x10\xc1\x02\x12\x0c\n\x07NANMEAN\x10\xc2\x02\x12\x12\n\rNANMEAN_CHUNK\x10\xc3\x02\x12\x0e\n\tNANARGMAX\x10\xc4\x02\x12\x14\n\x0fNANARGMAX_CHUNK\x10\xc5\x02\x12\x16\n\x11NANARGMAX_COMBINE\x10\xc6\x02\x12\x0e\n\tNANARGMIN\x10\xc7\x02\x12\x14\n\x0fNANARGMIN_CHUNK\x10\xc8\x02\x12\x16\n\x11NANARGMIN_COMBINE\x10\xc9\x02\x12\x12\n\rCOUNT_NONZERO\x10\xca\x02\x12\x11\n\x0cMOMENT_CHUNK\x10\xcb\x02\x12\x14\n\x0fNANMOMENT_CHUNK\x10\xcc\x02\x12\x13\n\x0eMOMENT_COMBINE\x10\xcd\x02\x12\x16\n\x11NANMOMENT_COMBINE\x10\xce\x02\x12\x0b\n\x06MOMENT\x10\xcf\x02\x12\x0e\n\tNANMOMENT\x10\xd0\x02\x12\x08\n\x03VAR\x10\xd1\x02\x12\x08\n\x03STD\x10\xd2\x02\x12\x0b\n\x06NANVAR\x10\xd3\x02\x12\x0b\n\x06NANSTD\x10\xd4\x02\x12\x0e\n\tNANCUMSUM\x10\xd5\x02\x12\x0f\n\nNANCUMPROD\x10\xd6\x02\x12\x0c\n\x07RESHAPE\x10\x91\x03\x12\n\n\x05SLICE\x10\x92\x03\x12\n\n\x05INDEX\x10\x93\x03\x12\x12\n\rINDEXSETVALUE\x10\x94\x03\x12\x10\n\x0b\x43ONCATENATE\x10\x95\x03\x12\x0c\n\x07RECHUNK\x10\x96\x03\x12\x0b\n\x06\x41STYPE\x10\x97\x03\x12\x0e\n\tTRANSPOSE\x10\x98\x03\x12\r\n\x08SWAPAXES\x10\x99\x03\x12\x11\n\x0c\x42ROADCAST_TO\x10\x9a\x03\x12\n\n\x05STACK\x10\x9b\x03\x12\n\n\x05WHERE\x10\x9c\x03\x12\x0b\n\x06\x43HOOSE\x10\x9d\x03\x12\x0c\n\x07NONZERO\x10\x9e\x03\x12\r\n\x08\x41RGWHERE\x10\x9f\x03\x12\x12\n\rUNRAVEL_INDEX\x10\xa0\x03\x12\x16\n\x11RAVEL_MULTI_INDEX\x10\xa1\x03\x12\x10\n\x0b\x41RRAY_SPLIT\x10\xa2\x03\x12\x0c\n\x07SQUEEZE\x10\xa3\x03\x12\r\n\x08\x44IGITIZE\x10\xa4\x03\x12\x0b\n\x06REPEAT\x10\xa5\x03\x12\x0b\n\x06\x43OPYTO\x10\xa6\x03\x12\t\n\x04ISIN\x10\xa7\x03\x12\x0e\n\tTENSORDOT\x10\xf5\x03\x12\x08\n\x03\x44OT\x10\xf6\x03\x12\x0b\n\x06MATMUL\x10\xf7\x03\x12\r\n\x08\x43HOLESKY\x10\xfe\x03\x12\x07\n\x02QR\x10\xff\x03\x12\x08\n\x03SVD\x10\x80\x04\x12\x07\n\x02LU\x10\x81\x04\x12\x15\n\x10SOLVE_TRIANGULAR\x10\x88\x04\x12\t\n\x04NORM\x10\x92\x04\x12\x08\n\x03\x46\x46T\x10\xd9\x04\x12\t\n\x04IFFT\x10\xda\x04\x12\t\n\x04\x46\x46T2\x10\xdb\x04\x12\n\n\x05IFFT2\x10\xdc\x04\x12\t\n\x04\x46\x46TN\x10\xdd\x04\x12\n\n\x05IFFTN\x10\xde\x04\x12\t\n\x04RFFT\x10\xdf\x04\x12\n\n\x05IRFFT\x10\xe0\x04\x12\n\n\x05RFFT2\x10\xe1\x04\x12\x0b\n\x06IRFFT2\x10\xe2\x04\x12\n\n\x05RFFTN\x10\xe3\x04\x12\x0b\n\x06IRFFTN\x10\xe4\x04\x12\t\n\x04HFFT\x10\xe5\x04\x12\n\n\x05IHFFT\x10\xe6\x04\x12\x0c\n\x07\x46\x46TFREQ\x10\xe7\x04\x12\x12\n\rFFTFREQ_CHUNK\x10\xe8\x04\x12\r\n\x08RFFTFREQ\x10\xe9\x04\x12\r\n\x08\x46\x46TSHIFT\x10\xea\x04\x12\x0e\n\tIFFTSHIFT\x10\xeb\x04\x12\x1e\n\x19SPARSE_MATRIX_DATA_SOURCE\x10\xbd\x05\x12\x14\n\x0f\x44\x45NSE_TO_SPARSE\x10\xbe\x05\x12\x14\n\x0fSPARSE_TO_DENSE\x10\xbf\x05\x12\t\n\x04\x46USE\x10\xa1\x06\x12\n\n\x05\x45NTER\x10\x85\x07\x12\n\n\x05LEAVE\x10\x86\x07\x12\x0f\n\nFIX_LATEST\x10\x87\x07\x12\x0c\n\x07IF_ELSE\x10\x88\x07\x12\x0e\n\tNEXT_ITER\x10\x89\x07\x12\x0e\n\tTABLE_COO\x10\xeb\x07\x12\x0e\n\tSTORE_COO\x10\xec\x07\x12\x0b\n\x05\x46\x45TCH\x10\xbf\x84=b\x06proto3') , dependencies=[mars_dot_serialize_dot_protos_dot_value__pb2.DESCRIPTOR,]) @@ -101,1174 +101,1182 @@ serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DATAFRAME_DATA_SOURCE', index=17, number=17, + name='TENSOR_FROM_TILEDB', index=17, number=18, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RAND', index=18, number=41, + name='TENSOR_STORE_TILEDB', index=18, number=19, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANDN', index=19, number=42, + name='DATAFRAME_DATA_SOURCE', index=19, number=17, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANDINT', index=20, number=43, + name='RAND_RAND', index=20, number=41, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANDOM_INTEGERS', index=21, number=44, + name='RAND_RANDN', index=21, number=42, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANDOM_SAMPLE', index=22, number=45, + name='RAND_RANDINT', index=22, number=43, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANDOM', index=23, number=46, + name='RAND_RANDOM_INTEGERS', index=23, number=44, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RANF', index=24, number=47, + name='RAND_RANDOM_SAMPLE', index=24, number=45, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_SAMPLE', index=25, number=48, + name='RAND_RANDOM', index=25, number=46, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_BYTES', index=26, number=49, + name='RAND_RANF', index=26, number=47, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_BETA', index=27, number=50, + name='RAND_SAMPLE', index=27, number=48, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_BINOMIAL', index=28, number=51, + name='RAND_BYTES', index=28, number=49, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_CHISQUARE', index=29, number=52, + name='RAND_BETA', index=29, number=50, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_CHOICE', index=30, number=53, + name='RAND_BINOMIAL', index=30, number=51, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_DIRICHLET', index=31, number=54, + name='RAND_CHISQUARE', index=31, number=52, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_EXPONENTIAL', index=32, number=55, + name='RAND_CHOICE', index=32, number=53, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_F', index=33, number=56, + name='RAND_DIRICHLET', index=33, number=54, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_GAMMA', index=34, number=57, + name='RAND_EXPONENTIAL', index=34, number=55, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_GEOMETRIC', index=35, number=58, + name='RAND_F', index=35, number=56, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_GUMBEL', index=36, number=59, + name='RAND_GAMMA', index=36, number=57, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_HYPERGEOMETRIC', index=37, number=60, + name='RAND_GEOMETRIC', index=37, number=58, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_LAPLACE', index=38, number=61, + name='RAND_GUMBEL', index=38, number=59, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_LOGISTIC', index=39, number=62, + name='RAND_HYPERGEOMETRIC', index=39, number=60, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_LOGNORMAL', index=40, number=63, + name='RAND_LAPLACE', index=40, number=61, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_LOGSERIES', index=41, number=64, + name='RAND_LOGISTIC', index=41, number=62, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_MULTINOMIAL', index=42, number=65, + name='RAND_LOGNORMAL', index=42, number=63, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_MULTIVARIATE_NORMAL', index=43, number=66, + name='RAND_LOGSERIES', index=43, number=64, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_NEGATIVE_BINOMIAL', index=44, number=67, + name='RAND_MULTINOMIAL', index=44, number=65, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_NONCENTRAL_CHISQURE', index=45, number=68, + name='RAND_MULTIVARIATE_NORMAL', index=45, number=66, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_NONCENTRAL_F', index=46, number=69, + name='RAND_NEGATIVE_BINOMIAL', index=46, number=67, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_NORMAL', index=47, number=70, + name='RAND_NONCENTRAL_CHISQURE', index=47, number=68, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_PARETO', index=48, number=71, + name='RAND_NONCENTRAL_F', index=48, number=69, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_PERMUTATION', index=49, number=72, + name='RAND_NORMAL', index=49, number=70, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_POSSION', index=50, number=73, + name='RAND_PARETO', index=50, number=71, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_POWER', index=51, number=74, + name='RAND_PERMUTATION', index=51, number=72, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_RAYLEIGH', index=52, number=75, + name='RAND_POSSION', index=52, number=73, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_SHUFFLE', index=53, number=76, + name='RAND_POWER', index=53, number=74, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_STANDARD_CAUCHY', index=54, number=77, + name='RAND_RAYLEIGH', index=54, number=75, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_STANDARD_EXPONENTIAL', index=55, number=78, + name='RAND_SHUFFLE', index=55, number=76, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_STANDARD_GAMMMA', index=56, number=79, + name='RAND_STANDARD_CAUCHY', index=56, number=77, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_STANDARD_NORMAL', index=57, number=80, + name='RAND_STANDARD_EXPONENTIAL', index=57, number=78, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_STANDARD_T', index=58, number=81, + name='RAND_STANDARD_GAMMMA', index=58, number=79, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_TOMAXINT', index=59, number=82, + name='RAND_STANDARD_NORMAL', index=59, number=80, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_TRIANGULAR', index=60, number=83, + name='RAND_STANDARD_T', index=60, number=81, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_UNIFORM', index=61, number=84, + name='RAND_TOMAXINT', index=61, number=82, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_VONMISES', index=62, number=85, + name='RAND_TRIANGULAR', index=62, number=83, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_WALD', index=63, number=86, + name='RAND_UNIFORM', index=63, number=84, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_WEIBULL', index=64, number=87, + name='RAND_VONMISES', index=64, number=85, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAND_ZIPF', index=65, number=88, + name='RAND_WALD', index=65, number=86, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ADD', index=66, number=101, + name='RAND_WEIBULL', index=66, number=87, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ADD_CONSTANT', index=67, number=102, + name='RAND_ZIPF', index=67, number=88, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SUB', index=68, number=103, + name='ADD', index=68, number=101, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SUB_CONSTANT', index=69, number=104, + name='ADD_CONSTANT', index=69, number=102, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MUL', index=70, number=105, + name='SUB', index=70, number=103, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MUL_CONSTANT', index=71, number=106, + name='SUB_CONSTANT', index=71, number=104, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DIV', index=72, number=107, + name='MUL', index=72, number=105, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DIV_CONSTANT', index=73, number=108, + name='MUL_CONSTANT', index=73, number=106, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TRUEDIV', index=74, number=109, + name='DIV', index=74, number=107, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TDIV_CONSTANT', index=75, number=110, + name='DIV_CONSTANT', index=75, number=108, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FLOORDIV', index=76, number=111, + name='TRUEDIV', index=76, number=109, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FDIV_CONSTANT', index=77, number=112, + name='TDIV_CONSTANT', index=77, number=110, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='POW', index=78, number=113, + name='FLOORDIV', index=78, number=111, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='POW_CONSTANT', index=79, number=114, + name='FDIV_CONSTANT', index=79, number=112, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MOD', index=80, number=115, + name='POW', index=80, number=113, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MOD_CONSTANT', index=81, number=116, + name='POW_CONSTANT', index=81, number=114, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMOD', index=82, number=117, + name='MOD', index=82, number=115, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMOD_CONSTANT', index=83, number=118, + name='MOD_CONSTANT', index=83, number=116, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOGADDEXP', index=84, number=119, + name='FMOD', index=84, number=117, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LAE_CONSTANT', index=85, number=120, + name='FMOD_CONSTANT', index=85, number=118, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOGADDEXP2', index=86, number=121, + name='LOGADDEXP', index=86, number=119, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LAE2_CONSTANT', index=87, number=122, + name='LAE_CONSTANT', index=87, number=120, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NEGATIVE', index=88, number=123, + name='LOGADDEXP2', index=88, number=121, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='POSITIVE', index=89, number=124, + name='LAE2_CONSTANT', index=89, number=122, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ABSOLUTE', index=90, number=125, + name='NEGATIVE', index=90, number=123, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FABS', index=91, number=126, + name='POSITIVE', index=91, number=124, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ABS', index=92, number=127, + name='ABSOLUTE', index=92, number=125, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RINT', index=93, number=128, + name='FABS', index=93, number=126, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SIGN', index=94, number=129, + name='ABS', index=94, number=127, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CONJ', index=95, number=130, + name='RINT', index=95, number=128, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='EXP', index=96, number=131, + name='SIGN', index=96, number=129, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='EXP2', index=97, number=132, + name='CONJ', index=97, number=130, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOG', index=98, number=133, + name='EXP', index=98, number=131, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOG2', index=99, number=134, + name='EXP2', index=99, number=132, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOG10', index=100, number=135, + name='LOG', index=100, number=133, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='EXPM1', index=101, number=136, + name='LOG2', index=101, number=134, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LOG1P', index=102, number=137, + name='LOG10', index=102, number=135, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SQRT', index=103, number=138, + name='EXPM1', index=103, number=136, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SQUARE', index=104, number=139, + name='LOG1P', index=104, number=137, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CBRT', index=105, number=140, + name='SQRT', index=105, number=138, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RECIPROCAL', index=106, number=141, + name='SQUARE', index=106, number=139, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='EQ', index=107, number=142, + name='CBRT', index=107, number=140, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='EQ_CONSTANT', index=108, number=143, + name='RECIPROCAL', index=108, number=141, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NE', index=109, number=144, + name='EQ', index=109, number=142, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NE_CONSTANT', index=110, number=145, + name='EQ_CONSTANT', index=110, number=143, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LT', index=111, number=146, + name='NE', index=111, number=144, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LT_CONSTANT', index=112, number=147, + name='NE_CONSTANT', index=112, number=145, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LE', index=113, number=148, + name='LT', index=113, number=146, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LE_CONSTANT', index=114, number=149, + name='LT_CONSTANT', index=114, number=147, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='GT', index=115, number=150, + name='LE', index=115, number=148, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='GT_CONSTANT', index=116, number=151, + name='LE_CONSTANT', index=116, number=149, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='GE', index=117, number=152, + name='GT', index=117, number=150, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='GE_CONSTANT', index=118, number=153, + name='GT_CONSTANT', index=118, number=151, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SIN', index=119, number=154, + name='GE', index=119, number=152, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COS', index=120, number=155, + name='GE_CONSTANT', index=120, number=153, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TAN', index=121, number=156, + name='SIN', index=121, number=154, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCSIN', index=122, number=157, + name='COS', index=122, number=155, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCCOS', index=123, number=158, + name='TAN', index=123, number=156, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCTAN', index=124, number=159, + name='ARCSIN', index=124, number=157, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCTAN2', index=125, number=160, + name='ARCCOS', index=125, number=158, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCT2_CONSTANT', index=126, number=161, + name='ARCTAN', index=126, number=159, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='HYPOT', index=127, number=162, + name='ARCTAN2', index=127, number=160, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='HYPOT_CONSTANT', index=128, number=163, + name='ARCT2_CONSTANT', index=128, number=161, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SINH', index=129, number=164, + name='HYPOT', index=129, number=162, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COSH', index=130, number=165, + name='HYPOT_CONSTANT', index=130, number=163, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TANH', index=131, number=166, + name='SINH', index=131, number=164, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCSINH', index=132, number=167, + name='COSH', index=132, number=165, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCCOSH', index=133, number=168, + name='TANH', index=133, number=166, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARCTANH', index=134, number=169, + name='ARCSINH', index=134, number=167, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DEG2RAD', index=135, number=170, + name='ARCCOSH', index=135, number=168, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAD2DEG', index=136, number=171, + name='ARCTANH', index=136, number=169, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITAND', index=137, number=172, + name='DEG2RAD', index=137, number=170, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITAND_CONSTANT', index=138, number=173, + name='RAD2DEG', index=138, number=171, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITOR', index=139, number=174, + name='BITAND', index=139, number=172, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITOR_CONSTANT', index=140, number=175, + name='BITAND_CONSTANT', index=140, number=173, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITXOR', index=141, number=176, + name='BITOR', index=141, number=174, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BITXOR_CONSTANT', index=142, number=177, + name='BITOR_CONSTANT', index=142, number=175, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='INVERT', index=143, number=178, + name='BITXOR', index=143, number=176, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LSHIFT', index=144, number=179, + name='BITXOR_CONSTANT', index=144, number=177, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LSHIFT_CONSTANT', index=145, number=180, + name='INVERT', index=145, number=178, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RSHIFT', index=146, number=181, + name='LSHIFT', index=146, number=179, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RSHIFT_CONSTANT', index=147, number=182, + name='LSHIFT_CONSTANT', index=147, number=180, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='AND', index=148, number=183, + name='RSHIFT', index=148, number=181, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='AND_CONSTANT', index=149, number=184, + name='RSHIFT_CONSTANT', index=149, number=182, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='OR', index=150, number=185, + name='AND', index=150, number=183, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='OR_CONSTANT', index=151, number=186, + name='AND_CONSTANT', index=151, number=184, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='XOR', index=152, number=187, + name='OR', index=152, number=185, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='XOR_CONSTANT', index=153, number=188, + name='OR_CONSTANT', index=153, number=186, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NOT', index=154, number=189, + name='XOR', index=154, number=187, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MAXIMUM', index=155, number=190, + name='XOR_CONSTANT', index=155, number=188, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MAXIMUM_CONSTANT', index=156, number=191, + name='NOT', index=156, number=189, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MINIMUM', index=157, number=192, + name='MAXIMUM', index=157, number=190, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MINIMUM_CONSTANT', index=158, number=193, + name='MAXIMUM_CONSTANT', index=158, number=191, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='AROUND', index=159, number=194, + name='MINIMUM', index=159, number=192, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FLOAT_POWER', index=160, number=195, + name='MINIMUM_CONSTANT', index=160, number=193, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FLOAT_POWER_CONSTANT', index=161, number=196, + name='AROUND', index=161, number=194, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMAX', index=162, number=197, + name='FLOAT_POWER', index=162, number=195, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMAX_CONSTANT', index=163, number=198, + name='FLOAT_POWER_CONSTANT', index=163, number=196, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMIN', index=164, number=199, + name='FMAX', index=164, number=197, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FMIN_CONSTANT', index=165, number=200, + name='FMAX_CONSTANT', index=165, number=198, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISFINITE', index=166, number=201, + name='FMIN', index=166, number=199, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISINF', index=167, number=202, + name='FMIN_CONSTANT', index=167, number=200, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISNAN', index=168, number=203, + name='ISFINITE', index=168, number=201, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SIGNBIT', index=169, number=204, + name='ISINF', index=169, number=202, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COPYSIGN', index=170, number=205, + name='ISNAN', index=170, number=203, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COPYSIGN_CONSTANT', index=171, number=206, + name='SIGNBIT', index=171, number=204, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NEXTAFTER', index=172, number=207, + name='COPYSIGN', index=172, number=205, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NEXTAFTER_CONSTANT', index=173, number=208, + name='COPYSIGN_CONSTANT', index=173, number=206, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SPACING', index=174, number=209, + name='NEXTAFTER', index=174, number=207, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LDEXP', index=175, number=210, + name='NEXTAFTER_CONSTANT', index=175, number=208, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LDEXP_CONSTANT', index=176, number=211, + name='SPACING', index=176, number=209, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FREXP', index=177, number=212, + name='LDEXP', index=177, number=210, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MODF', index=178, number=213, + name='LDEXP_CONSTANT', index=178, number=211, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FLOOR', index=179, number=214, + name='FREXP', index=179, number=212, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CEIL', index=180, number=215, + name='MODF', index=180, number=213, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TRUNC', index=181, number=216, + name='FLOOR', index=181, number=214, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DEGREES', index=182, number=217, + name='CEIL', index=182, number=215, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RADIANS', index=183, number=218, + name='TRUNC', index=183, number=216, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CLIP', index=184, number=219, + name='DEGREES', index=184, number=217, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISREAL', index=185, number=220, + name='RADIANS', index=185, number=218, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISCOMPLEX', index=186, number=221, + name='CLIP', index=186, number=219, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='REAL', index=187, number=222, + name='ISREAL', index=187, number=220, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IMAG', index=188, number=223, + name='ISCOMPLEX', index=188, number=221, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FIX', index=189, number=224, + name='REAL', index=189, number=222, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='I0', index=190, number=225, + name='IMAG', index=190, number=223, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SINC', index=191, number=226, + name='FIX', index=191, number=224, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NAN_TO_NUM', index=192, number=227, + name='I0', index=192, number=225, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISCLOSE', index=193, number=228, + name='SINC', index=193, number=226, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISCLOSE_CONSTANT', index=194, number=229, + name='NAN_TO_NUM', index=194, number=227, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DIVMOD', index=195, number=231, + name='ISCLOSE', index=195, number=228, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ANGLE', index=196, number=232, + name='ISCLOSE_CONSTANT', index=196, number=229, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SET_REAL', index=197, number=233, + name='DIVMOD', index=197, number=231, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SET_REAL_CONSTANT', index=198, number=234, + name='ANGLE', index=198, number=232, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SET_IMAG', index=199, number=235, + name='SET_REAL', index=199, number=233, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SET_IMAG_CONSTANT', index=200, number=236, + name='SET_REAL_CONSTANT', index=200, number=234, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TREE_ADD', index=201, number=251, + name='SET_IMAG', index=201, number=235, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TREE_MULTIPLY', index=202, number=252, + name='SET_IMAG_CONSTANT', index=202, number=236, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CUMSUM', index=203, number=301, + name='TREE_ADD', index=203, number=251, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CUMPROD', index=204, number=302, + name='TREE_MULTIPLY', index=204, number=252, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='PROD', index=205, number=303, + name='CUMSUM', index=205, number=301, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SUM', index=206, number=304, + name='CUMPROD', index=206, number=302, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MAX', index=207, number=305, + name='PROD', index=207, number=303, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MIN', index=208, number=306, + name='SUM', index=208, number=304, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ALL', index=209, number=307, + name='MAX', index=209, number=305, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ANY', index=210, number=308, + name='MIN', index=210, number=306, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MEAN_CHUNK', index=211, number=309, + name='ALL', index=211, number=307, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MEAN_COMBINE', index=212, number=310, + name='ANY', index=212, number=308, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MEAN', index=213, number=311, + name='MEAN_CHUNK', index=213, number=309, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMAX', index=214, number=312, + name='MEAN_COMBINE', index=214, number=310, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMAX_CHUNK', index=215, number=313, + name='MEAN', index=215, number=311, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMAX_COMBINE', index=216, number=314, + name='ARGMAX', index=216, number=312, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMIN', index=217, number=315, + name='ARGMAX_CHUNK', index=217, number=313, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMIN_CHUNK', index=218, number=316, + name='ARGMAX_COMBINE', index=218, number=314, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGMIN_COMBINE', index=219, number=317, + name='ARGMIN', index=219, number=315, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANSUM', index=220, number=318, + name='ARGMIN_CHUNK', index=220, number=316, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMAX', index=221, number=319, + name='ARGMIN_COMBINE', index=221, number=317, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMIN', index=222, number=320, + name='NANSUM', index=222, number=318, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANPROD', index=223, number=321, + name='NANMAX', index=223, number=319, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMEAN', index=224, number=322, + name='NANMIN', index=224, number=320, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMEAN_CHUNK', index=225, number=323, + name='NANPROD', index=225, number=321, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMAX', index=226, number=324, + name='NANMEAN', index=226, number=322, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMAX_CHUNK', index=227, number=325, + name='NANMEAN_CHUNK', index=227, number=323, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMAX_COMBINE', index=228, number=326, + name='NANARGMAX', index=228, number=324, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMIN', index=229, number=327, + name='NANARGMAX_CHUNK', index=229, number=325, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMIN_CHUNK', index=230, number=328, + name='NANARGMAX_COMBINE', index=230, number=326, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANARGMIN_COMBINE', index=231, number=329, + name='NANARGMIN', index=231, number=327, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COUNT_NONZERO', index=232, number=330, + name='NANARGMIN_CHUNK', index=232, number=328, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MOMENT_CHUNK', index=233, number=331, + name='NANARGMIN_COMBINE', index=233, number=329, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMOMENT_CHUNK', index=234, number=332, + name='COUNT_NONZERO', index=234, number=330, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MOMENT_COMBINE', index=235, number=333, + name='MOMENT_CHUNK', index=235, number=331, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMOMENT_COMBINE', index=236, number=334, + name='NANMOMENT_CHUNK', index=236, number=332, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MOMENT', index=237, number=335, + name='MOMENT_COMBINE', index=237, number=333, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANMOMENT', index=238, number=336, + name='NANMOMENT_COMBINE', index=238, number=334, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='VAR', index=239, number=337, + name='MOMENT', index=239, number=335, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='STD', index=240, number=338, + name='NANMOMENT', index=240, number=336, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANVAR', index=241, number=339, + name='VAR', index=241, number=337, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANSTD', index=242, number=340, + name='STD', index=242, number=338, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANCUMSUM', index=243, number=341, + name='NANVAR', index=243, number=339, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NANCUMPROD', index=244, number=342, + name='NANSTD', index=244, number=340, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RESHAPE', index=245, number=401, + name='NANCUMSUM', index=245, number=341, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SLICE', index=246, number=402, + name='NANCUMPROD', index=246, number=342, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='INDEX', index=247, number=403, + name='RESHAPE', index=247, number=401, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='INDEXSETVALUE', index=248, number=404, + name='SLICE', index=248, number=402, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CONCATENATE', index=249, number=405, + name='INDEX', index=249, number=403, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RECHUNK', index=250, number=406, + name='INDEXSETVALUE', index=250, number=404, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ASTYPE', index=251, number=407, + name='CONCATENATE', index=251, number=405, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TRANSPOSE', index=252, number=408, + name='RECHUNK', index=252, number=406, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SWAPAXES', index=253, number=409, + name='ASTYPE', index=253, number=407, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='BROADCAST_TO', index=254, number=410, + name='TRANSPOSE', index=254, number=408, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='STACK', index=255, number=411, + name='SWAPAXES', index=255, number=409, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='WHERE', index=256, number=412, + name='BROADCAST_TO', index=256, number=410, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CHOOSE', index=257, number=413, + name='STACK', index=257, number=411, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NONZERO', index=258, number=414, + name='WHERE', index=258, number=412, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARGWHERE', index=259, number=415, + name='CHOOSE', index=259, number=413, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='UNRAVEL_INDEX', index=260, number=416, + name='NONZERO', index=260, number=414, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RAVEL_MULTI_INDEX', index=261, number=417, + name='ARGWHERE', index=261, number=415, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ARRAY_SPLIT', index=262, number=418, + name='UNRAVEL_INDEX', index=262, number=416, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SQUEEZE', index=263, number=419, + name='RAVEL_MULTI_INDEX', index=263, number=417, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DIGITIZE', index=264, number=420, + name='ARRAY_SPLIT', index=264, number=418, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='REPEAT', index=265, number=421, + name='SQUEEZE', index=265, number=419, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='COPYTO', index=266, number=422, + name='DIGITIZE', index=266, number=420, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ISIN', index=267, number=423, + name='REPEAT', index=267, number=421, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TENSORDOT', index=268, number=501, + name='COPYTO', index=268, number=422, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DOT', index=269, number=502, + name='ISIN', index=269, number=423, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='MATMUL', index=270, number=503, + name='TENSORDOT', index=270, number=501, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='CHOLESKY', index=271, number=510, + name='DOT', index=271, number=502, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='QR', index=272, number=511, + name='MATMUL', index=272, number=503, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SVD', index=273, number=512, + name='CHOLESKY', index=273, number=510, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LU', index=274, number=513, + name='QR', index=274, number=511, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SOLVE_TRIANGULAR', index=275, number=520, + name='SVD', index=275, number=512, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NORM', index=276, number=530, + name='LU', index=276, number=513, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFT', index=277, number=601, + name='SOLVE_TRIANGULAR', index=277, number=520, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IFFT', index=278, number=602, + name='NORM', index=278, number=530, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFT2', index=279, number=603, + name='FFT', index=279, number=601, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IFFT2', index=280, number=604, + name='IFFT', index=280, number=602, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFTN', index=281, number=605, + name='FFT2', index=281, number=603, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IFFTN', index=282, number=606, + name='IFFT2', index=282, number=604, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RFFT', index=283, number=607, + name='FFTN', index=283, number=605, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IRFFT', index=284, number=608, + name='IFFTN', index=284, number=606, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RFFT2', index=285, number=609, + name='RFFT', index=285, number=607, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IRFFT2', index=286, number=610, + name='IRFFT', index=286, number=608, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RFFTN', index=287, number=611, + name='RFFT2', index=287, number=609, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IRFFTN', index=288, number=612, + name='IRFFT2', index=288, number=610, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='HFFT', index=289, number=613, + name='RFFTN', index=289, number=611, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IHFFT', index=290, number=614, + name='IRFFTN', index=290, number=612, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFTFREQ', index=291, number=615, + name='HFFT', index=291, number=613, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFTFREQ_CHUNK', index=292, number=616, + name='IHFFT', index=292, number=614, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='RFFTFREQ', index=293, number=617, + name='FFTFREQ', index=293, number=615, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FFTSHIFT', index=294, number=618, + name='FFTFREQ_CHUNK', index=294, number=616, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IFFTSHIFT', index=295, number=619, + name='RFFTFREQ', index=295, number=617, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SPARSE_MATRIX_DATA_SOURCE', index=296, number=701, + name='FFTSHIFT', index=296, number=618, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='DENSE_TO_SPARSE', index=297, number=702, + name='IFFTSHIFT', index=297, number=619, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='SPARSE_TO_DENSE', index=298, number=703, + name='SPARSE_MATRIX_DATA_SOURCE', index=298, number=701, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FUSE', index=299, number=801, + name='DENSE_TO_SPARSE', index=299, number=702, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='ENTER', index=300, number=901, + name='SPARSE_TO_DENSE', index=300, number=703, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='LEAVE', index=301, number=902, + name='FUSE', index=301, number=801, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FIX_LATEST', index=302, number=903, + name='ENTER', index=302, number=901, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='IF_ELSE', index=303, number=904, + name='LEAVE', index=303, number=902, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='NEXT_ITER', index=304, number=905, + name='FIX_LATEST', index=304, number=903, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='TABLE_COO', index=305, number=1003, + name='IF_ELSE', index=305, number=904, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='STORE_COO', index=306, number=1004, + name='NEXT_ITER', index=306, number=905, serialized_options=None, type=None), _descriptor.EnumValueDescriptor( - name='FETCH', index=307, number=999999, + name='TABLE_COO', index=307, number=1003, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='STORE_COO', index=308, number=1004, + serialized_options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FETCH', index=309, number=999999, serialized_options=None, type=None), ], containing_type=None, serialized_options=None, serialized_start=237, - serialized_end=5097, + serialized_end=5146, ) _sym_db.RegisterEnumDescriptor(_OPERANDDEF_OPERANDTYPE) @@ -1359,7 +1367,7 @@ oneofs=[ ], serialized_start=75, - serialized_end=5097, + serialized_end=5146, ) _OPERANDDEF_ATTRENTRY.fields_by_name['value'].message_type = mars_dot_serialize_dot_protos_dot_value__pb2._VALUE diff --git a/mars/tensor/__init__.py b/mars/tensor/__init__.py index f4eed52002..9082282112 100644 --- a/mars/tensor/__init__.py +++ b/mars/tensor/__init__.py @@ -18,7 +18,8 @@ from .expressions.datasource import tensor, array, asarray, scalar, \ empty, empty_like, ones, ones_like, zeros, zeros_like, \ full, arange, diag, diagflat, eye, identity, linspace, \ - meshgrid, indices, tril, triu + meshgrid, indices, tril, triu, fromtiledb +from .expressions.datastore import totiledb from .expressions.base import result_type, copyto, transpose, where, broadcast_to, broadcast_arrays, \ expand_dims, rollaxis, swapaxes, moveaxis, ravel, atleast_1d, atleast_2d, atleast_3d, argwhere, \ array_split, split, hsplit, vsplit, dsplit, roll, squeeze, ptp, diff, ediff1d, digitize, \ diff --git a/mars/tensor/core.py b/mars/tensor/core.py index 93d5891d55..0d2231a025 100644 --- a/mars/tensor/core.py +++ b/mars/tensor/core.py @@ -173,6 +173,11 @@ def ravel(self): def _equals(self, o): return self is o + def totiledb(self, uri, ctx=None, key=None, timestamp=None): + from .expressions.datastore import totiledb + + return totiledb(uri, self, ctx=ctx, key=key, timestamp=timestamp) + def execute(self, session=None, **kw): from ..session import Session @@ -361,6 +366,11 @@ def reshape(self, shape, *shapes): """ return self._data.reshape(shape, *shapes) + def totiledb(self, uri, ctx=None, key=None, timestamp=None): + from .expressions.datastore import totiledb + + return totiledb(uri, self, ctx=ctx, key=key, timestamp=timestamp) + class SparseTensor(Tensor): __slots__ = () diff --git a/mars/tensor/execution/core.py b/mars/tensor/execution/core.py index 28ee60229d..a94beb2198 100644 --- a/mars/tensor/execution/core.py +++ b/mars/tensor/execution/core.py @@ -15,6 +15,7 @@ # limitations under the License. from .datasource import register_data_source_handler +from .datastore import register_data_store_handler from .random import register_random_handler from .base import register_basic_handler from .arithmetic import register_arithmetic_handler @@ -48,6 +49,7 @@ def register_tensor_execution_handler(): from .cp import register_cp_handler register_cp_handler() register_data_source_handler() + register_data_store_handler() register_random_handler() register_basic_handler() register_arithmetic_handler() diff --git a/mars/tensor/execution/datasource.py b/mars/tensor/execution/datasource.py index 341e148edd..811ba81f1c 100644 --- a/mars/tensor/execution/datasource.py +++ b/mars/tensor/execution/datasource.py @@ -21,6 +21,7 @@ from ...lib.sparse.core import get_sparse_module, get_array_module, cps, sps, naked from ...lib.sparse import SparseNDArray from ..expressions import datasource +from .utils import get_tiledb_ctx logger = logging.getLogger(__name__) @@ -155,6 +156,52 @@ def _tensor_dense_to_sparse(ctx, chunk): ctx[chunk.key] = SparseNDArray(xps.csr_matrix(in_data)) +def _tensor_tiledb(ctx, chunk): + import tiledb + + xp = array_module(chunk.op.gpu) + + axis_offsets = [offset + dim_start for offset, dim_start + in zip(chunk.op.axis_offsets, chunk.op.tiledb_dim_starts)] + tiledb_ctx = get_tiledb_ctx(chunk.op.tiledb_config) + uri = chunk.op.tiledb_uri + key = chunk.op.tiledb_key + timestamp = chunk.op.tiledb_timestamp + + slcs = [] + for axis in range(chunk.ndim): + axis_offset = axis_offsets[axis] + axis_length = chunk.shape[axis] + slcs.append(slice(axis_offset, axis_offset + axis_length)) + + if not chunk.issparse(): + # read dense array from tiledb + with tiledb.DenseArray(tiledb_ctx, uri, key=key, timestamp=timestamp) as tiledb_arr: + ctx[chunk.key] = tiledb_arr[tuple(slcs)] + else: + # read sparse array from tiledb + with tiledb.SparseArray(tiledb_ctx, uri, key=key, timestamp=timestamp) as tiledb_arr: + if tiledb_arr.ndim > 2: + raise NotImplementedError( + 'Does not support to read array with more than 2 dimensions') + + data = tiledb_arr[tuple(slcs)] + coords = data['coords'] + value = data[tiledb_arr.attr(0).name] + if tiledb_arr.ndim == 2: + # 2-d + ij = tuple(coords[tiledb_arr.domain.dim(k).name] - axis_offsets[k] + for k in range(tiledb_arr.ndim)) + spmatrix = sps.coo_matrix((value, ij), shape=chunk.shape) + ctx[chunk.key] = SparseNDArray(spmatrix) + else: + # 1-d + ij = xp.zeros(coords.shape), \ + coords[tiledb_arr.domain.dim(0).name] - axis_offsets[0] + spmatrix = sps.coo_matrix((value, ij), shape=(1,) + chunk.shape) + ctx[chunk.key] = SparseNDArray(spmatrix, shape=chunk.shape) + + def _tensor_fetch_chunk(ctx, chunk): # nothing need to do return @@ -187,5 +234,7 @@ def register_data_source_handler(): register(datasource.CSRMatrixDataSource, _tensor_csr_matrix_data_source) register(datasource.SparseToDense, _tensor_sparse_to_dense) register(datasource.DenseToSparse, _tensor_dense_to_sparse) + register(datasource.TensorTileDBDataSource, _tensor_tiledb) register(datasource.TensorFetch, _tensor_fetch_chunk) register(datasource.Scalar, _scalar) + diff --git a/mars/tensor/execution/datastore.py b/mars/tensor/execution/datastore.py index 260ffce827..e402affb74 100644 --- a/mars/tensor/execution/datastore.py +++ b/mars/tensor/execution/datastore.py @@ -14,10 +14,55 @@ # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None -def _data_store(ctx, chunk): - raise NotImplementedError +from ...lib.sparse import SparseNDArray +from ...lib.sparse.core import sps +from ..expressions import datastore +from .utils import get_tiledb_ctx + + +def _store_tiledb(ctx, chunk): + tiledb_ctx = get_tiledb_ctx(chunk.op.tiledb_config) + uri = chunk.op.tiledb_uri + key = chunk.op.tiledb_key + timestamp = chunk.op.tiledb_timestamp + axis_offsets = chunk.op.axis_offsets + + if not chunk.issparse(): + # dense + to_store = np.ascontiguousarray(ctx[chunk.op.input.key]) + slcs = [] + for axis in range(chunk.ndim): + axis_offset = axis_offsets[axis] + axis_length = chunk.op.input.shape[axis] + slcs.append(slice(axis_offset, axis_offset + axis_length)) + with tiledb.DenseArray(tiledb_ctx, uri, mode='w', + key=key, timestamp=timestamp) as arr: + arr[tuple(slcs)] = to_store + ctx[chunk.key] = np.empty((0,) * chunk.ndim, dtype=chunk.dtype) + else: + # sparse + to_store = ctx[chunk.op.input.key].spmatrix.tocoo() + if to_store.nnz > 0: + with tiledb.SparseArray(tiledb_ctx, uri, mode='w', + key=key, timestamp=timestamp) as arr: + if chunk.ndim == 1: + vec = to_store.col if to_store.shape[0] == 1 else to_store.row + vec += axis_offsets[0] + arr[vec] = to_store.data + else: + i, j = to_store.row + axis_offsets[0], to_store.col + axis_offsets[1] + arr[i, j] = to_store.data + ctx[chunk.key] = SparseNDArray(sps.csr_matrix((0, 0), dtype=chunk.dtype), + shape=chunk.shape) def register_data_store_handler(): - pass + from ...executor import register + + register(datastore.TensorTileDBDataStore, _store_tiledb) diff --git a/mars/tensor/execution/tests/test_datasource_execute.py b/mars/tensor/execution/tests/test_datasource_execute.py index ad919bb197..965a244557 100644 --- a/mars/tensor/execution/tests/test_datasource_execute.py +++ b/mars/tensor/execution/tests/test_datasource_execute.py @@ -14,14 +14,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import tempfile +import shutil +import unittest + import numpy as np import scipy.sparse as sps +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None from mars.executor import Executor from mars.tests.core import TestBase from mars.tensor.expressions.datasource import tensor, ones_like, zeros, zeros_like, full, \ arange, empty, empty_like, diag, diagflat, eye, linspace, meshgrid, indices, \ - triu, tril + triu, tril, fromtiledb from mars.lib.sparse import SparseNDArray from mars.tensor.expressions.lib import nd_grid @@ -652,3 +660,74 @@ def testIndexTrickExecution(self): expected = np.lib.index_tricks.nd_grid(sparse=True)[0:5, 0:5] [np.testing.assert_equal(r, e) for r, e in zip(res, expected)] + @unittest.skipIf(tiledb is None, 'tiledb not installed') + def testReadTileDBExecution(self): + ctx = tiledb.Ctx() + + tempdir = tempfile.mkdtemp() + try: + # create TileDB dense array + dom = tiledb.Domain(ctx, + tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32), + tiledb.Dim(ctx, domain=(0, 90), tile=22, dtype=np.int32), + tiledb.Dim(ctx, domain=(0, 9), tile=8, dtype=np.int32), + ) + schema = tiledb.ArraySchema(ctx, domain=dom, sparse=False, + attrs=[tiledb.Attr(ctx, dtype=np.float64)]) + tiledb.DenseArray.create(tempdir, schema) + + expected = np.random.rand(100, 91, 10) + with tiledb.DenseArray(ctx, tempdir, mode='w') as arr: + arr.write_direct(expected) + + a = fromtiledb(tempdir, ctx=ctx) + result = self.executor.execute_tensor(a, concat=True)[0] + + np.testing.assert_allclose(expected, result) + finally: + shutil.rmtree(tempdir) + + tempdir = tempfile.mkdtemp() + try: + # create 2-d TileDB sparse array + dom = tiledb.Domain(ctx, + tiledb.Dim(ctx, domain=(0, 99), tile=30, dtype=np.int32), + tiledb.Dim(ctx, domain=(2, 11), tile=8, dtype=np.int32), + ) + schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True, + attrs=[tiledb.Attr(ctx, dtype=np.float64)]) + tiledb.SparseArray.create(tempdir, schema) + + expected = sps.rand(100, 10, density=0.01) + with tiledb.SparseArray(ctx, tempdir, mode='w') as arr: + I, J = expected.row, expected.col + 2 + arr[I, J] = {arr.attr(0).name: expected.data} + + a = fromtiledb(tempdir, ctx=ctx) + result = self.executor.execute_tensor(a, concat=True)[0] + + np.testing.assert_allclose(expected.toarray(), result.toarray()) + finally: + shutil.rmtree(tempdir) + + tempdir = tempfile.mkdtemp() + try: + # create 1-d TileDB sparse array + dom = tiledb.Domain(ctx, + tiledb.Dim(ctx, domain=(1, 100), tile=30, dtype=np.int32), + ) + schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True, + attrs=[tiledb.Attr(ctx, dtype=np.float64)]) + tiledb.SparseArray.create(tempdir, schema) + + expected = sps.rand(1, 100, density=0.05) + with tiledb.SparseArray(ctx, tempdir, mode='w') as arr: + I= expected.col + 1 + arr[I] = expected.data + + a = fromtiledb(tempdir, ctx=ctx) + result = self.executor.execute_tensor(a, concat=True)[0] + + np.testing.assert_allclose(expected.toarray()[0], result.toarray()) + finally: + shutil.rmtree(tempdir) diff --git a/mars/tensor/execution/tests/test_datastore_execute.py b/mars/tensor/execution/tests/test_datastore_execute.py new file mode 100644 index 0000000000..013aca5e4c --- /dev/null +++ b/mars/tensor/execution/tests/test_datastore_execute.py @@ -0,0 +1,70 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import shutil +import unittest + +import numpy as np +import scipy.sparse as sps +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None + +from mars.executor import Executor +from mars.tests.core import TestBase +from mars.tensor import tensor, totiledb + + +class Test(TestBase): + def setUp(self): + super(Test, self).setUp() + self.executor = Executor() + + @unittest.skipIf(tiledb is None, 'tiledb not installed') + def testStoreTileDBExecution(self): + ctx = tiledb.Ctx() + + tempdir = tempfile.mkdtemp() + try: + # store TileDB dense array + expected = np.random.rand(8, 4, 3) + a = tensor(expected, chunk_size=(3, 3, 2)) + save = totiledb(tempdir, a, ctx=ctx) + self.executor.execute_tensor(save) + + with tiledb.DenseArray(ctx, tempdir) as arr: + np.testing.assert_allclose(expected, arr.read_direct()) + finally: + shutil.rmtree(tempdir) + + tempdir = tempfile.mkdtemp() + try: + # store 2-d TileDB sparse array + expected = sps.random(8, 7, density=0.1) + a = tensor(expected, chunk_size=(3, 5)) + save = totiledb(tempdir, a, ctx=ctx) + self.executor.execute_tensor(save) + + with tiledb.SparseArray(ctx, tempdir) as arr: + data = arr[:, :] + coords = data['coords'] + value = data[arr.attr(0).name] + ij = tuple(coords[arr.domain.dim(k).name] for k in range(arr.ndim)) + result = sps.coo_matrix((value, ij), shape=arr.shape) + + np.testing.assert_allclose(expected.toarray(), result.toarray()) + finally: + shutil.rmtree(tempdir) diff --git a/mars/tensor/execution/utils.py b/mars/tensor/execution/utils.py new file mode 100644 index 0000000000..cbfe1cbcec --- /dev/null +++ b/mars/tensor/execution/utils.py @@ -0,0 +1,35 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import tiledb +except ImportError: # pragma: no cover + tildb = None + +from ...compat import functools32 + + +# As TileDB Ctx's creation is a bit time-consuming, +# we just cache the Ctx +# also remember the arguments should be hashable +@functools32.lru_cache(10) +def _create_tiledb_ctx(conf_tuple): + if conf_tuple is not None: + return tiledb.Ctx(dict(conf_tuple)) + return tiledb.Ctx() + + +def get_tiledb_ctx(conf): + key = tuple(conf.items()) if conf is not None else None + return _create_tiledb_ctx(key) diff --git a/mars/tensor/expressions/datasource/__init__.py b/mars/tensor/expressions/datasource/__init__.py index 6ee7fc4e2a..9fd33d9e73 100644 --- a/mars/tensor/expressions/datasource/__init__.py +++ b/mars/tensor/expressions/datasource/__init__.py @@ -32,4 +32,5 @@ from .tri import triu, tril, TensorTriu, TensorTril from .fromdense import fromdense, DenseToSparse from .fromsparse import fromsparse, SparseToDense +from .fromtiledb import fromtiledb, TensorTileDBDataSource from .core import TensorFetch diff --git a/mars/tensor/expressions/datasource/fromtiledb.py b/mars/tensor/expressions/datasource/fromtiledb.py new file mode 100644 index 0000000000..6c6021fa4d --- /dev/null +++ b/mars/tensor/expressions/datasource/fromtiledb.py @@ -0,0 +1,110 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .... import opcodes as OperandDef +from ....serialize import ValueType, DictField, TupleField, StringField, Int64Field +from .core import TensorNoInput + + +class TensorTileDBDataSource(TensorNoInput): + _op_type_ = OperandDef.TENSOR_FROM_TILEDB + + _tiledb_config = DictField('tiledb_config') + # URI of array to open + _tiledb_uri = StringField('tiledb_uri') + # tiledb dim start + _tiledb_dim_starts = TupleField('tiledb_dim_starts', ValueType.int64) + # encryption key to decrypt if provided + _tiledb_key = StringField('tiledb_key') + # open array at a given timestamp if provided + _tiledb_timestamp = Int64Field('tiledb_timestamp') + _axis_offsets = TupleField('axis_offsets', ValueType.int64) + + def __init__(self, tiledb_config=None, tiledb_uri=None, tiledb_dim_starts=None, + tiledb_key=None, tiledb_timstamp=None, dtype=None, + gpu=None, sparse=None, **kw): + super(TensorTileDBDataSource, self).__init__( + _tiledb_config=tiledb_config, _tiledb_uri=tiledb_uri, + _tiledb_dim_starts=tiledb_dim_starts, + _tiledb_key=tiledb_key, _tiledb_timestamp=tiledb_timstamp, + _dtype=dtype, _gpu=gpu, _sparse=sparse, **kw) + + @property + def tiledb_config(self): + return self._tiledb_config + + @property + def tiledb_uri(self): + return self._tiledb_uri + + @property + def tiledb_dim_starts(self): + return self._tiledb_dim_starts + + @property + def tiledb_key(self): + return self._tiledb_key + + @property + def tiledb_timestamp(self): + return self._tiledb_timestamp + + @property + def axis_offsets(self): + return self._axis_offsets + + def to_chunk_op(self, *args): + _, chunk_idx, nsplits = args + chunk_op = super(TensorTileDBDataSource, self).to_chunk_op(*args) + axis_offsets = [] + for axis, idx in enumerate(chunk_idx): + axis_offsets.append(sum(nsplits[axis][:idx])) + chunk_op._axis_offsets = tuple(axis_offsets) + return chunk_op + + +def fromtiledb(uri, ctx=None, key=None, timestamp=None, gpu=None): + import tiledb + + raw_ctx = ctx + if raw_ctx is None: + ctx = tiledb.Ctx() + + # get metadata from tiledb + try: + tiledb_arr = tiledb.DenseArray(ctx, uri, key=key, timestamp=timestamp) + sparse = False + except ValueError: + # if the array is not dense, ValueError will be raised by tiledb + tiledb_arr = tiledb.SparseArray(ctx, uri, key=key, timestamp=timestamp) + sparse = True + + if tiledb_arr.nattr > 1: + raise NotImplementedError('Does not supported TileDB array schema ' + 'with more than 1 attr') + tiledb_dim_starts = tuple(tiledb_arr.domain.dim(j).domain[0] + for j in range(tiledb_arr.ndim)) + if any(isinstance(s, float) for s in tiledb_dim_starts): + raise ValueError('Does not support TileDB array schema ' + 'whose dimensions has float domain') + + dtype = tiledb_arr.attr(0).dtype + tiledb_config = None if raw_ctx is None else ctx.config().dict() + op = TensorTileDBDataSource(tiledb_config=tiledb_config, tiledb_uri=uri, + tiledb_key=key, tiledb_timstamp=timestamp, + tiledb_dim_starts=tiledb_dim_starts, + gpu=gpu, sparse=sparse, dtype=dtype) + chunk_size = tuple(int(tiledb_arr.domain.dim(i).tile) + for i in range(tiledb_arr.domain.ndim)) + return op(tiledb_arr.shape, chunk_size=chunk_size) diff --git a/mars/tensor/expressions/datastore/__init__.py b/mars/tensor/expressions/datastore/__init__.py index 1d409cc542..8b31f11e9e 100644 --- a/mars/tensor/expressions/datastore/__init__.py +++ b/mars/tensor/expressions/datastore/__init__.py @@ -12,4 +12,6 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +from .totiledb import totiledb, TensorTileDBDataStore diff --git a/mars/tensor/expressions/datastore/core.py b/mars/tensor/expressions/datastore/core.py new file mode 100644 index 0000000000..b28aa77b5b --- /dev/null +++ b/mars/tensor/expressions/datastore/core.py @@ -0,0 +1,50 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ....operands import DataStore +from ..core import TensorOperandMixin + + +class TensorDataStore(DataStore, TensorOperandMixin): + def _set_inputs(self, inputs): + super(TensorDataStore, self)._set_inputs(inputs) + self._input = inputs[0] + + def __call__(self, a): + shape = (0,) * a.ndim + return self.new_tensor([a], shape) + + def calc_shape(self, *inputs_shape): + return self.outputs[0].shape + + @classmethod + def _get_out_chunk(cls, op, in_chunk): + chunk_op = op.copy().reset_key() + out_chunk_shape = (0,) * in_chunk.ndim + return chunk_op.new_chunk([in_chunk], out_chunk_shape, + index=in_chunk.index) + + @classmethod + def tile(cls, op): + in_tensor = op.input + + out_chunks = [] + for chunk in in_tensor.chunks: + out_chunk = cls._get_out_chunk(op, chunk) + out_chunks.append(out_chunk) + + new_op = op.copy() + return new_op.new_tensors(op.inputs, op.outputs[0].shape, + chunks=out_chunks, + nsplits=((0,) * len(ns) for ns in in_tensor.nsplits)) diff --git a/mars/tensor/expressions/datastore/totiledb.py b/mars/tensor/expressions/datastore/totiledb.py new file mode 100644 index 0000000000..d5e88f1d22 --- /dev/null +++ b/mars/tensor/expressions/datastore/totiledb.py @@ -0,0 +1,114 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .... import opcodes as OperandDef +from ....serialize import ValueType, DictField, TupleField, StringField, Int64Field, KeyField +from ..datasource import tensor as astensor +from .core import TensorDataStore +from .utils import check_tiledb_array_with_tensor, get_tiledb_schema_from_tensor + + +class TensorTileDBDataStore(TensorDataStore): + _op_type_ = OperandDef.TENSOR_STORE_TILEDB + + _input = KeyField('input') + _tiledb_config = DictField('tiledb_config') + # URI of array to write + _tiledb_uri = StringField('tiledb_uri') + # encryption key to decrypt if provided + _tiledb_key = StringField('tiledb_key') + # open array at a given timestamp if provided + _tiledb_timestamp = Int64Field('tiledb_timestamp') + _axis_offsets = TupleField('axis_offsets', ValueType.int64) + + def __init__(self, tiledb_config=None, tiledb_uri=None, tiledb_key=None, + tiledb_timestamp=None, dtype=None, sparse=None, **kw): + super(TensorTileDBDataStore, self).__init__( + _tiledb_config=tiledb_config, _tiledb_uri=tiledb_uri, _tiledb_key=tiledb_key, + _tiledb_timestamp=tiledb_timestamp, _dtype=dtype, _sparse=sparse, **kw) + + @property + def tiledb_config(self): + return self._tiledb_config + + @property + def tiledb_uri(self): + return self._tiledb_uri + + @property + def tiledb_key(self): + return self._tiledb_key + + @property + def tiledb_timestamp(self): + return self._tiledb_timestamp + + @property + def axis_offsets(self): + return self._axis_offsets + + @classmethod + def _get_out_chunk(cls, op, in_chunk): + chunk_op = op.copy().reset_key() + nsplits = op.input.nsplits + axis_offsets = [] + for axis, idx in enumerate(in_chunk.index): + axis_offsets.append(sum(nsplits[axis][:idx])) + chunk_op._axis_offsets = tuple(axis_offsets) + out_chunk_shape = (0,) * in_chunk.ndim + return chunk_op.new_chunk([in_chunk], out_chunk_shape, + index=in_chunk.index) + + @classmethod + def tile(cls, op): + import tiledb + + tensor = super(TensorTileDBDataStore, cls).tile(op)[0] + + ctx = tiledb.Ctx(op.tiledb_config) + tiledb_array_type = tiledb.SparseArray if tensor.issparse() else tiledb.DenseArray + try: + tiledb_array_type(ctx, op.tiledb_uri, key=op.tiledb_key, + timestamp=op.tiledb_timestamp) + except tiledb.TileDBError: + # not exist, try to create TileDB Array by given uri + tiledb_array_schema = get_tiledb_schema_from_tensor(op.input, ctx, op.input.nsplits) + tiledb_array_type.create(op.tiledb_uri, tiledb_array_schema, key=op.tiledb_key) + + return [tensor] + + +def totiledb(uri, x, ctx=None, key=None, timestamp=None): + import tiledb + + x = astensor(x) + raw_ctx = ctx + if raw_ctx is None: + ctx = tiledb.Ctx() + + tiledb_array_type = tiledb.SparseArray if x.issparse() else tiledb.DenseArray + try: + tiledb_array = tiledb_array_type(ctx, uri, key=key, timestamp=timestamp) + # if already created, we will check the shape and dtype + check_tiledb_array_with_tensor(x, tiledb_array) + except tiledb.TileDBError: + # not exist, as we don't know the tile, + # we will create the tiledb array in the tile of tensor + pass + + tiledb_config = None if raw_ctx is None else raw_ctx.config().dict() + op = TensorTileDBDataStore(tiledb_config=tiledb_config, tiledb_uri=uri, + tiledb_key=key, tiledb_timestamp=timestamp, + dtype=x.dtype, sparse=x.issparse()) + return op(x) diff --git a/mars/tensor/expressions/datastore/utils.py b/mars/tensor/expressions/datastore/utils.py new file mode 100644 index 0000000000..c70692c813 --- /dev/null +++ b/mars/tensor/expressions/datastore/utils.py @@ -0,0 +1,49 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None + + +def get_tiledb_schema_from_tensor(tensor, tiledb_ctx, nsplits, **kw): + ctx = tiledb_ctx + + dims = [] + for d in range(tensor.ndim): + extent = tensor.shape[d] + domain = (0, extent - 1) + tile = max(nsplits[d]) + dims.append(tiledb.Dim(ctx, "", domain, tile=tile, dtype=np.int64)) + dom = tiledb.Domain(ctx, *dims) + att = tiledb.Attr(ctx, dtype=tensor.dtype) + return tiledb.ArraySchema(ctx, domain=dom, attrs=(att,), + sparse=tensor.issparse(), **kw) + + +def check_tiledb_array_with_tensor(tensor, tiledb_array): + if tensor.ndim != tiledb_array.ndim: + # ndim + raise ValueError('ndim of TileDB Array to store is different to tensor, ' + 'expect {0}, got {1}'.format(tensor.ndim, tiledb_array.ndim)) + if tensor.shape != tiledb_array.shape: + # shape + raise ValueError('shape of TileDB Array to store is different to tensor, ' + 'expect {0}, got {1}'.format(tensor.shape, tiledb_array.shape)) + if tensor.dtype != tiledb_array.attr(0).dtype: + # dtype + raise ValueError('dtype of TileDB Array to store is different to tensor, ' + 'expect {0}, got {1}'.format(tensor.dtype, tiledb_array.domain.dtype)) diff --git a/mars/tensor/expressions/tests/test_core.py b/mars/tensor/expressions/tests/test_core.py index cf69b2fa8a..f4b87a670c 100644 --- a/mars/tensor/expressions/tests/test_core.py +++ b/mars/tensor/expressions/tests/test_core.py @@ -23,7 +23,7 @@ from mars.tensor import ones, zeros, tensor, full, arange, diag, linspace, triu, tril, ones_like, dot from mars.tensor.expressions.datasource import fromdense from mars.tensor.expressions.datasource.tri import TensorTriu, TensorTril -from mars.tensor.expressions.datasource.zeros import zeros, TensorZeros +from mars.tensor.expressions.datasource.zeros import TensorZeros from mars.tensor.expressions.datasource.fromdense import DenseToSparse from mars.tensor.expressions.datasource.array import CSRMatrixDataSource from mars.tensor.expressions.datasource.ones import TensorOnes, TensorOnesLike diff --git a/mars/tensor/expressions/tests/test_datasource.py b/mars/tensor/expressions/tests/test_datasource.py new file mode 100644 index 0000000000..d3d223e444 --- /dev/null +++ b/mars/tensor/expressions/tests/test_datasource.py @@ -0,0 +1,106 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import shutil +import tempfile + +import numpy as np +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None + +from mars.tests.core import TestBase +from mars.tensor.expressions.datasource import fromtiledb, TensorTileDBDataSource + + +class Test(TestBase): + @unittest.skipIf(tiledb is None, 'TileDB not installed') + def testFromTileDB(self): + ctx = tiledb.Ctx() + + for sparse in (True, False): + dom = tiledb.Domain( + ctx, + tiledb.Dim(ctx, name="i", domain=(1, 30), tile=7, dtype=np.int32), + tiledb.Dim(ctx, name="j", domain=(1, 20), tile=3, dtype=np.int32), + tiledb.Dim(ctx, name="k", domain=(1, 10), tile=4, dtype=np.int32), + ) + schema = tiledb.ArraySchema(ctx, domain=dom, sparse=sparse, + attrs=[tiledb.Attr(ctx, name='a', dtype=np.float32)]) + + tempdir = tempfile.mkdtemp() + try: + # create tiledb array + tiledb.DenseArray.create(tempdir, schema) + + tensor = fromtiledb(tempdir) + self.assertIsInstance(tensor.op, TensorTileDBDataSource) + self.assertEqual(tensor.op.issparse(), sparse) + self.assertEqual(tensor.shape, (30, 20, 10)) + self.assertEqual(tensor.params.raw_chunk_size, (7, 3, 4)) + self.assertIsNone(tensor.op.tiledb_config) + self.assertEqual(tensor.op.tiledb_uri, tempdir) + self.assertIsNone(tensor.op.tiledb_key) + self.assertIsNone(tensor.op.tiledb_timestamp) + + tensor.tiles() + + self.assertEqual(len(tensor.chunks), 105) + self.assertIsInstance(tensor.chunks[0].op, TensorTileDBDataSource) + self.assertEqual(tensor.chunks[0].op.issparse(), sparse) + self.assertEqual(tensor.chunks[0].shape, (7, 3, 4)) + self.assertIsNone(tensor.chunks[0].op.tiledb_config) + self.assertEqual(tensor.chunks[0].op.tiledb_uri, tempdir) + self.assertIsNone(tensor.chunks[0].op.tiledb_key) + self.assertIsNone(tensor.chunks[0].op.tiledb_timestamp) + self.assertEqual(tensor.chunks[0].op.tiledb_dim_starts, (1, 1, 1)) + + # test axis_offsets of chunk op + self.assertEqual(tensor.chunks[0].op.axis_offsets, (0, 0, 0)) + self.assertEqual(tensor.chunks[1].op.axis_offsets, (0, 0, 4)) + self.assertEqual(tensor.cix[0, 2, 2].op.axis_offsets, (0, 6, 8)) + self.assertEqual(tensor.cix[0, 6, 2].op.axis_offsets, (0, 18, 8)) + self.assertEqual(tensor.cix[4, 6, 2].op.axis_offsets, (28, 18, 8)) + + tensor2 = fromtiledb(tempdir, ctx=ctx) + self.assertEqual(tensor2.op.tiledb_config, ctx.config().dict()) + + tensor2.tiles() + + self.assertEqual(tensor2.chunks[0].op.tiledb_config, ctx.config().dict()) + finally: + shutil.rmtree(tempdir) + + @unittest.skipIf(tiledb is None, 'TileDB not installed') + def testDimStartFloat(self): + ctx = tiledb.Ctx() + + dom = tiledb.Domain( + ctx, + tiledb.Dim(ctx, name="i", domain=(0.0, 6.0), tile=6, dtype=np.float64), + ) + schema = tiledb.ArraySchema(ctx, domain=dom, sparse=True, + attrs=[tiledb.Attr(ctx, name='a', dtype=np.float32)]) + + tempdir = tempfile.mkdtemp() + try: + # create tiledb array + tiledb.SparseArray.create(tempdir, schema) + + with self.assertRaises(ValueError): + fromtiledb(tempdir, ctx=ctx) + finally: + shutil.rmtree(tempdir) diff --git a/mars/tensor/expressions/tests/test_datastore.py b/mars/tensor/expressions/tests/test_datastore.py new file mode 100644 index 0000000000..f8a8fc4a01 --- /dev/null +++ b/mars/tensor/expressions/tests/test_datastore.py @@ -0,0 +1,103 @@ +# Copyright 1999-2018 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import shutil +import tempfile + +import numpy as np +try: + import tiledb +except ImportError: # pragma: no cover + tiledb = None + +from mars.tensor import random +from mars.tensor.expressions.datastore.utils import get_tiledb_schema_from_tensor, \ + check_tiledb_array_with_tensor +from mars.tensor.expressions.datastore import totiledb +from mars.tests.core import TestBase + + +class Test(TestBase): + @unittest.skipIf(tiledb is None, 'TileDB not installed') + def testGetTileDBSchema(self): + ctx = tiledb.Ctx() + + nsplits = ((1, 2), (3, 1), (2, 2, 1)) + a = random.rand(3, 4, 5, dtype=np.float64, chunk_size=nsplits) + schema = get_tiledb_schema_from_tensor(a, ctx, nsplits) + self.assertEqual(schema.ndim, 3) + self.assertEqual(schema.shape, (3, 4, 5)) + self.assertEqual([schema.domain.dim(i).tile for i in range(a.ndim)], [2, 3, 2]) + self.assertEqual(schema.attr(0).dtype, a.dtype) + + @unittest.skipIf(tiledb is None, 'TileDB not installed') + def testCheckTileDB(self): + ctx = tiledb.Ctx() + + tempdir = tempfile.mkdtemp() + try: + np_a = np.random.rand(2, 3) + tiledb_a = tiledb.DenseArray.from_numpy(ctx, tempdir, np_a) + + with self.assertRaises(ValueError): + # ndim not match + check_tiledb_array_with_tensor(random.rand(2, 3, 4), tiledb_a) + + with self.assertRaises(ValueError): + # shape not matchn + check_tiledb_array_with_tensor(random.rand(2, 4), tiledb_a) + + with self.assertRaises(ValueError): + # dtype not match + check_tiledb_array_with_tensor(random.rand(2, 3, dtype=np.float32), tiledb_a) + + # legal + check_tiledb_array_with_tensor(random.rand(2, 3), tiledb_a) + finally: + shutil.rmtree(tempdir) + + @unittest.skipIf(tiledb is None, 'TileDB not installed') + def testStoreTileDB(self): + ctx = tiledb.Ctx() + tempdir = tempfile.mkdtemp() + try: + t = random.rand(50, 30, chunk_size=13) + t2 = t + 1 + + saved = totiledb(tempdir, t2) + self.assertEqual(saved.shape, (0, 0)) + self.assertIsNone(saved.op.tiledb_config) + self.assertEquals(saved.op.tiledb_uri, tempdir) + + with self.assertRaises(tiledb.TileDBError): + tiledb.DenseArray(ctx, tempdir) + + # tiledb array is created in the tile + saved.tiles() + + # no error + tiledb.DenseArray(ctx, tempdir) + + self.assertEqual(saved.chunks[0].op.axis_offsets, (0, 0)) + self.assertEqual(saved.chunks[1].op.axis_offsets, (0, 13)) + self.assertEqual(saved.cix[0, 2].op.axis_offsets, (0, 26)) + self.assertEqual(saved.cix[1, 2].op.axis_offsets, (13, 26)) + self.assertEqual(saved.cix[3, 2].op.axis_offsets, (39, 26)) + + with self.assertRaises(ValueError): + t3 = random.rand(30, 50) + totiledb(tempdir, t3, ctx=ctx) # shape incompatible + finally: + shutil.rmtree(tempdir)