From c910cbdf07012a931eb0d0303b4b61b1f980ab32 Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Tue, 4 Nov 2025 08:30:52 +0100 Subject: [PATCH] Initial commit --- python/pyspark/sql/connect/proto/types_pb2.py | 116 +++++++++--------- .../pyspark/sql/connect/proto/types_pb2.pyi | 59 +++++++++ .../main/protobuf/spark/connect/types.proto | 18 ++- 3 files changed, 134 insertions(+), 59 deletions(-) diff --git a/python/pyspark/sql/connect/proto/types_pb2.py b/python/pyspark/sql/connect/proto/types_pb2.py index 9a52129103ad..4e35f6b8911a 100644 --- a/python/pyspark/sql/connect/proto/types_pb2.py +++ b/python/pyspark/sql/connect/proto/types_pb2.py @@ -35,7 +35,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b"\n\x19spark/connect/types.proto\x12\rspark.connect\"\xac#\n\x08\x44\x61taType\x12\x32\n\x04null\x18\x01 \x01(\x0b\x32\x1c.spark.connect.DataType.NULLH\x00R\x04null\x12\x38\n\x06\x62inary\x18\x02 \x01(\x0b\x32\x1e.spark.connect.DataType.BinaryH\x00R\x06\x62inary\x12;\n\x07\x62oolean\x18\x03 \x01(\x0b\x32\x1f.spark.connect.DataType.BooleanH\x00R\x07\x62oolean\x12\x32\n\x04\x62yte\x18\x04 \x01(\x0b\x32\x1c.spark.connect.DataType.ByteH\x00R\x04\x62yte\x12\x35\n\x05short\x18\x05 \x01(\x0b\x32\x1d.spark.connect.DataType.ShortH\x00R\x05short\x12;\n\x07integer\x18\x06 \x01(\x0b\x32\x1f.spark.connect.DataType.IntegerH\x00R\x07integer\x12\x32\n\x04long\x18\x07 \x01(\x0b\x32\x1c.spark.connect.DataType.LongH\x00R\x04long\x12\x35\n\x05\x66loat\x18\x08 \x01(\x0b\x32\x1d.spark.connect.DataType.FloatH\x00R\x05\x66loat\x12\x38\n\x06\x64ouble\x18\t \x01(\x0b\x32\x1e.spark.connect.DataType.DoubleH\x00R\x06\x64ouble\x12;\n\x07\x64\x65\x63imal\x18\n \x01(\x0b\x32\x1f.spark.connect.DataType.DecimalH\x00R\x07\x64\x65\x63imal\x12\x38\n\x06string\x18\x0b \x01(\x0b\x32\x1e.spark.connect.DataType.StringH\x00R\x06string\x12\x32\n\x04\x63har\x18\x0c \x01(\x0b\x32\x1c.spark.connect.DataType.CharH\x00R\x04\x63har\x12<\n\x08var_char\x18\r \x01(\x0b\x32\x1f.spark.connect.DataType.VarCharH\x00R\x07varChar\x12\x32\n\x04\x64\x61te\x18\x0e \x01(\x0b\x32\x1c.spark.connect.DataType.DateH\x00R\x04\x64\x61te\x12\x41\n\ttimestamp\x18\x0f \x01(\x0b\x32!.spark.connect.DataType.TimestampH\x00R\ttimestamp\x12K\n\rtimestamp_ntz\x18\x10 \x01(\x0b\x32$.spark.connect.DataType.TimestampNTZH\x00R\x0ctimestampNtz\x12W\n\x11\x63\x61lendar_interval\x18\x11 \x01(\x0b\x32(.spark.connect.DataType.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12[\n\x13year_month_interval\x18\x12 \x01(\x0b\x32).spark.connect.DataType.YearMonthIntervalH\x00R\x11yearMonthInterval\x12U\n\x11\x64\x61y_time_interval\x18\x13 \x01(\x0b\x32'.spark.connect.DataType.DayTimeIntervalH\x00R\x0f\x64\x61yTimeInterval\x12\x35\n\x05\x61rray\x18\x14 \x01(\x0b\x32\x1d.spark.connect.DataType.ArrayH\x00R\x05\x61rray\x12\x38\n\x06struct\x18\x15 \x01(\x0b\x32\x1e.spark.connect.DataType.StructH\x00R\x06struct\x12/\n\x03map\x18\x16 \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x03map\x12;\n\x07variant\x18\x19 \x01(\x0b\x32\x1f.spark.connect.DataType.VariantH\x00R\x07variant\x12/\n\x03udt\x18\x17 \x01(\x0b\x32\x1b.spark.connect.DataType.UDTH\x00R\x03udt\x12>\n\x08unparsed\x18\x18 \x01(\x0b\x32 .spark.connect.DataType.UnparsedH\x00R\x08unparsed\x12\x32\n\x04time\x18\x1c \x01(\x0b\x32\x1c.spark.connect.DataType.TimeH\x00R\x04time\x1a\x43\n\x07\x42oolean\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x42yte\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x41\n\x05Short\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x43\n\x07Integer\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04Long\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x41\n\x05\x46loat\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x42\n\x06\x44ouble\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a`\n\x06String\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x12\x1c\n\tcollation\x18\x02 \x01(\tR\tcollation\x1a\x42\n\x06\x42inary\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04NULL\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x45\n\tTimestamp\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x44\x61te\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aH\n\x0cTimestampNTZ\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aq\n\x04Time\x12!\n\tprecision\x18\x01 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReferenceB\x0c\n\n_precision\x1aL\n\x10\x43\x61lendarInterval\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\xb3\x01\n\x11YearMonthInterval\x12$\n\x0bstart_field\x18\x01 \x01(\x05H\x00R\nstartField\x88\x01\x01\x12 \n\tend_field\x18\x02 \x01(\x05H\x01R\x08\x65ndField\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x0e\n\x0c_start_fieldB\x0c\n\n_end_field\x1a\xb1\x01\n\x0f\x44\x61yTimeInterval\x12$\n\x0bstart_field\x18\x01 \x01(\x05H\x00R\nstartField\x88\x01\x01\x12 \n\tend_field\x18\x02 \x01(\x05H\x01R\x08\x65ndField\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x0e\n\x0c_start_fieldB\x0c\n\n_end_field\x1aX\n\x04\x43har\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a[\n\x07VarChar\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\x99\x01\n\x07\x44\x65\x63imal\x12\x19\n\x05scale\x18\x01 \x01(\x05H\x00R\x05scale\x88\x01\x01\x12!\n\tprecision\x18\x02 \x01(\x05H\x01R\tprecision\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x08\n\x06_scaleB\x0c\n\n_precision\x1a\xa1\x01\n\x0bStructField\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x34\n\tdata_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x12\x1a\n\x08nullable\x18\x03 \x01(\x08R\x08nullable\x12\x1f\n\x08metadata\x18\x04 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x7f\n\x06Struct\x12;\n\x06\x66ields\x18\x01 \x03(\x0b\x32#.spark.connect.DataType.StructFieldR\x06\x66ields\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\xa2\x01\n\x05\x41rray\x12:\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x0b\x65lementType\x12#\n\rcontains_null\x18\x02 \x01(\x08R\x0c\x63ontainsNull\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReference\x1a\xdb\x01\n\x03Map\x12\x32\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x07keyType\x12\x36\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\tvalueType\x12.\n\x13value_contains_null\x18\x03 \x01(\x08R\x11valueContainsNull\x12\x38\n\x18type_variation_reference\x18\x04 \x01(\rR\x16typeVariationReference\x1a\x43\n\x07Variant\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\xa1\x02\n\x03UDT\x12\x12\n\x04type\x18\x01 \x01(\tR\x04type\x12 \n\tjvm_class\x18\x02 \x01(\tH\x00R\x08jvmClass\x88\x01\x01\x12&\n\x0cpython_class\x18\x03 \x01(\tH\x01R\x0bpythonClass\x88\x01\x01\x12;\n\x17serialized_python_class\x18\x04 \x01(\tH\x02R\x15serializedPythonClass\x88\x01\x01\x12\x37\n\x08sql_type\x18\x05 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x03R\x07sqlType\x88\x01\x01\x42\x0c\n\n_jvm_classB\x0f\n\r_python_classB\x1a\n\x18_serialized_python_classB\x0b\n\t_sql_type\x1a\x34\n\x08Unparsed\x12(\n\x10\x64\x61ta_type_string\x18\x01 \x01(\tR\x0e\x64\x61taTypeStringB\x06\n\x04kindJ\x04\x08\x1a\x10\x1bJ\x04\x08\x1b\x10\x1c\x42\x36\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3" + b"\n\x19spark/connect/types.proto\x12\rspark.connect\"\xd8%\n\x08\x44\x61taType\x12\x32\n\x04null\x18\x01 \x01(\x0b\x32\x1c.spark.connect.DataType.NULLH\x00R\x04null\x12\x38\n\x06\x62inary\x18\x02 \x01(\x0b\x32\x1e.spark.connect.DataType.BinaryH\x00R\x06\x62inary\x12;\n\x07\x62oolean\x18\x03 \x01(\x0b\x32\x1f.spark.connect.DataType.BooleanH\x00R\x07\x62oolean\x12\x32\n\x04\x62yte\x18\x04 \x01(\x0b\x32\x1c.spark.connect.DataType.ByteH\x00R\x04\x62yte\x12\x35\n\x05short\x18\x05 \x01(\x0b\x32\x1d.spark.connect.DataType.ShortH\x00R\x05short\x12;\n\x07integer\x18\x06 \x01(\x0b\x32\x1f.spark.connect.DataType.IntegerH\x00R\x07integer\x12\x32\n\x04long\x18\x07 \x01(\x0b\x32\x1c.spark.connect.DataType.LongH\x00R\x04long\x12\x35\n\x05\x66loat\x18\x08 \x01(\x0b\x32\x1d.spark.connect.DataType.FloatH\x00R\x05\x66loat\x12\x38\n\x06\x64ouble\x18\t \x01(\x0b\x32\x1e.spark.connect.DataType.DoubleH\x00R\x06\x64ouble\x12;\n\x07\x64\x65\x63imal\x18\n \x01(\x0b\x32\x1f.spark.connect.DataType.DecimalH\x00R\x07\x64\x65\x63imal\x12\x38\n\x06string\x18\x0b \x01(\x0b\x32\x1e.spark.connect.DataType.StringH\x00R\x06string\x12\x32\n\x04\x63har\x18\x0c \x01(\x0b\x32\x1c.spark.connect.DataType.CharH\x00R\x04\x63har\x12<\n\x08var_char\x18\r \x01(\x0b\x32\x1f.spark.connect.DataType.VarCharH\x00R\x07varChar\x12\x32\n\x04\x64\x61te\x18\x0e \x01(\x0b\x32\x1c.spark.connect.DataType.DateH\x00R\x04\x64\x61te\x12\x41\n\ttimestamp\x18\x0f \x01(\x0b\x32!.spark.connect.DataType.TimestampH\x00R\ttimestamp\x12K\n\rtimestamp_ntz\x18\x10 \x01(\x0b\x32$.spark.connect.DataType.TimestampNTZH\x00R\x0ctimestampNtz\x12W\n\x11\x63\x61lendar_interval\x18\x11 \x01(\x0b\x32(.spark.connect.DataType.CalendarIntervalH\x00R\x10\x63\x61lendarInterval\x12[\n\x13year_month_interval\x18\x12 \x01(\x0b\x32).spark.connect.DataType.YearMonthIntervalH\x00R\x11yearMonthInterval\x12U\n\x11\x64\x61y_time_interval\x18\x13 \x01(\x0b\x32'.spark.connect.DataType.DayTimeIntervalH\x00R\x0f\x64\x61yTimeInterval\x12\x35\n\x05\x61rray\x18\x14 \x01(\x0b\x32\x1d.spark.connect.DataType.ArrayH\x00R\x05\x61rray\x12\x38\n\x06struct\x18\x15 \x01(\x0b\x32\x1e.spark.connect.DataType.StructH\x00R\x06struct\x12/\n\x03map\x18\x16 \x01(\x0b\x32\x1b.spark.connect.DataType.MapH\x00R\x03map\x12;\n\x07variant\x18\x19 \x01(\x0b\x32\x1f.spark.connect.DataType.VariantH\x00R\x07variant\x12/\n\x03udt\x18\x17 \x01(\x0b\x32\x1b.spark.connect.DataType.UDTH\x00R\x03udt\x12>\n\x08geometry\x18\x1a \x01(\x0b\x32 .spark.connect.DataType.GeometryH\x00R\x08geometry\x12\x41\n\tgeography\x18\x1b \x01(\x0b\x32!.spark.connect.DataType.GeographyH\x00R\tgeography\x12>\n\x08unparsed\x18\x18 \x01(\x0b\x32 .spark.connect.DataType.UnparsedH\x00R\x08unparsed\x12\x32\n\x04time\x18\x1c \x01(\x0b\x32\x1c.spark.connect.DataType.TimeH\x00R\x04time\x1a\x43\n\x07\x42oolean\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x42yte\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x41\n\x05Short\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x43\n\x07Integer\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04Long\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x41\n\x05\x46loat\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x42\n\x06\x44ouble\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a`\n\x06String\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x12\x1c\n\tcollation\x18\x02 \x01(\tR\tcollation\x1a\x42\n\x06\x42inary\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04NULL\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\x45\n\tTimestamp\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a@\n\x04\x44\x61te\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aH\n\x0cTimestampNTZ\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1aq\n\x04Time\x12!\n\tprecision\x18\x01 \x01(\x05H\x00R\tprecision\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReferenceB\x0c\n\n_precision\x1aL\n\x10\x43\x61lendarInterval\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\xb3\x01\n\x11YearMonthInterval\x12$\n\x0bstart_field\x18\x01 \x01(\x05H\x00R\nstartField\x88\x01\x01\x12 \n\tend_field\x18\x02 \x01(\x05H\x01R\x08\x65ndField\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x0e\n\x0c_start_fieldB\x0c\n\n_end_field\x1a\xb1\x01\n\x0f\x44\x61yTimeInterval\x12$\n\x0bstart_field\x18\x01 \x01(\x05H\x00R\nstartField\x88\x01\x01\x12 \n\tend_field\x18\x02 \x01(\x05H\x01R\x08\x65ndField\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x0e\n\x0c_start_fieldB\x0c\n\n_end_field\x1aX\n\x04\x43har\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a[\n\x07VarChar\x12\x16\n\x06length\x18\x01 \x01(\x05R\x06length\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\x99\x01\n\x07\x44\x65\x63imal\x12\x19\n\x05scale\x18\x01 \x01(\x05H\x00R\x05scale\x88\x01\x01\x12!\n\tprecision\x18\x02 \x01(\x05H\x01R\tprecision\x88\x01\x01\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReferenceB\x08\n\x06_scaleB\x0c\n\n_precision\x1a\xa1\x01\n\x0bStructField\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x34\n\tdata_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x08\x64\x61taType\x12\x1a\n\x08nullable\x18\x03 \x01(\x08R\x08nullable\x12\x1f\n\x08metadata\x18\x04 \x01(\tH\x00R\x08metadata\x88\x01\x01\x42\x0b\n\t_metadata\x1a\x7f\n\x06Struct\x12;\n\x06\x66ields\x18\x01 \x03(\x0b\x32#.spark.connect.DataType.StructFieldR\x06\x66ields\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\xa2\x01\n\x05\x41rray\x12:\n\x0c\x65lement_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x0b\x65lementType\x12#\n\rcontains_null\x18\x02 \x01(\x08R\x0c\x63ontainsNull\x12\x38\n\x18type_variation_reference\x18\x03 \x01(\rR\x16typeVariationReference\x1a\xdb\x01\n\x03Map\x12\x32\n\x08key_type\x18\x01 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x07keyType\x12\x36\n\nvalue_type\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\tvalueType\x12.\n\x13value_contains_null\x18\x03 \x01(\x08R\x11valueContainsNull\x12\x38\n\x18type_variation_reference\x18\x04 \x01(\rR\x16typeVariationReference\x1aX\n\x08Geometry\x12\x12\n\x04srid\x18\x01 \x01(\x05R\x04srid\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1aY\n\tGeography\x12\x12\n\x04srid\x18\x01 \x01(\x05R\x04srid\x12\x38\n\x18type_variation_reference\x18\x02 \x01(\rR\x16typeVariationReference\x1a\x43\n\x07Variant\x12\x38\n\x18type_variation_reference\x18\x01 \x01(\rR\x16typeVariationReference\x1a\xa1\x02\n\x03UDT\x12\x12\n\x04type\x18\x01 \x01(\tR\x04type\x12 \n\tjvm_class\x18\x02 \x01(\tH\x00R\x08jvmClass\x88\x01\x01\x12&\n\x0cpython_class\x18\x03 \x01(\tH\x01R\x0bpythonClass\x88\x01\x01\x12;\n\x17serialized_python_class\x18\x04 \x01(\tH\x02R\x15serializedPythonClass\x88\x01\x01\x12\x37\n\x08sql_type\x18\x05 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x03R\x07sqlType\x88\x01\x01\x42\x0c\n\n_jvm_classB\x0f\n\r_python_classB\x1a\n\x18_serialized_python_classB\x0b\n\t_sql_type\x1a\x34\n\x08Unparsed\x12(\n\x10\x64\x61ta_type_string\x18\x01 \x01(\tR\x0e\x64\x61taTypeStringB\x06\n\x04kindB6\n\x1eorg.apache.spark.connect.protoP\x01Z\x12internal/generatedb\x06proto3" ) _globals = globals() @@ -47,59 +47,63 @@ "DESCRIPTOR" ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated" _globals["_DATATYPE"]._serialized_start = 45 - _globals["_DATATYPE"]._serialized_end = 4569 - _globals["_DATATYPE_BOOLEAN"]._serialized_start = 1647 - _globals["_DATATYPE_BOOLEAN"]._serialized_end = 1714 - _globals["_DATATYPE_BYTE"]._serialized_start = 1716 - _globals["_DATATYPE_BYTE"]._serialized_end = 1780 - _globals["_DATATYPE_SHORT"]._serialized_start = 1782 - _globals["_DATATYPE_SHORT"]._serialized_end = 1847 - _globals["_DATATYPE_INTEGER"]._serialized_start = 1849 - _globals["_DATATYPE_INTEGER"]._serialized_end = 1916 - _globals["_DATATYPE_LONG"]._serialized_start = 1918 - _globals["_DATATYPE_LONG"]._serialized_end = 1982 - _globals["_DATATYPE_FLOAT"]._serialized_start = 1984 - _globals["_DATATYPE_FLOAT"]._serialized_end = 2049 - _globals["_DATATYPE_DOUBLE"]._serialized_start = 2051 - _globals["_DATATYPE_DOUBLE"]._serialized_end = 2117 - _globals["_DATATYPE_STRING"]._serialized_start = 2119 - _globals["_DATATYPE_STRING"]._serialized_end = 2215 - _globals["_DATATYPE_BINARY"]._serialized_start = 2217 - _globals["_DATATYPE_BINARY"]._serialized_end = 2283 - _globals["_DATATYPE_NULL"]._serialized_start = 2285 - _globals["_DATATYPE_NULL"]._serialized_end = 2349 - _globals["_DATATYPE_TIMESTAMP"]._serialized_start = 2351 - _globals["_DATATYPE_TIMESTAMP"]._serialized_end = 2420 - _globals["_DATATYPE_DATE"]._serialized_start = 2422 - _globals["_DATATYPE_DATE"]._serialized_end = 2486 - _globals["_DATATYPE_TIMESTAMPNTZ"]._serialized_start = 2488 - _globals["_DATATYPE_TIMESTAMPNTZ"]._serialized_end = 2560 - _globals["_DATATYPE_TIME"]._serialized_start = 2562 - _globals["_DATATYPE_TIME"]._serialized_end = 2675 - _globals["_DATATYPE_CALENDARINTERVAL"]._serialized_start = 2677 - _globals["_DATATYPE_CALENDARINTERVAL"]._serialized_end = 2753 - _globals["_DATATYPE_YEARMONTHINTERVAL"]._serialized_start = 2756 - _globals["_DATATYPE_YEARMONTHINTERVAL"]._serialized_end = 2935 - _globals["_DATATYPE_DAYTIMEINTERVAL"]._serialized_start = 2938 - _globals["_DATATYPE_DAYTIMEINTERVAL"]._serialized_end = 3115 - _globals["_DATATYPE_CHAR"]._serialized_start = 3117 - _globals["_DATATYPE_CHAR"]._serialized_end = 3205 - _globals["_DATATYPE_VARCHAR"]._serialized_start = 3207 - _globals["_DATATYPE_VARCHAR"]._serialized_end = 3298 - _globals["_DATATYPE_DECIMAL"]._serialized_start = 3301 - _globals["_DATATYPE_DECIMAL"]._serialized_end = 3454 - _globals["_DATATYPE_STRUCTFIELD"]._serialized_start = 3457 - _globals["_DATATYPE_STRUCTFIELD"]._serialized_end = 3618 - _globals["_DATATYPE_STRUCT"]._serialized_start = 3620 - _globals["_DATATYPE_STRUCT"]._serialized_end = 3747 - _globals["_DATATYPE_ARRAY"]._serialized_start = 3750 - _globals["_DATATYPE_ARRAY"]._serialized_end = 3912 - _globals["_DATATYPE_MAP"]._serialized_start = 3915 - _globals["_DATATYPE_MAP"]._serialized_end = 4134 - _globals["_DATATYPE_VARIANT"]._serialized_start = 4136 - _globals["_DATATYPE_VARIANT"]._serialized_end = 4203 - _globals["_DATATYPE_UDT"]._serialized_start = 4206 - _globals["_DATATYPE_UDT"]._serialized_end = 4495 - _globals["_DATATYPE_UNPARSED"]._serialized_start = 4497 - _globals["_DATATYPE_UNPARSED"]._serialized_end = 4549 + _globals["_DATATYPE"]._serialized_end = 4869 + _globals["_DATATYPE_BOOLEAN"]._serialized_start = 1778 + _globals["_DATATYPE_BOOLEAN"]._serialized_end = 1845 + _globals["_DATATYPE_BYTE"]._serialized_start = 1847 + _globals["_DATATYPE_BYTE"]._serialized_end = 1911 + _globals["_DATATYPE_SHORT"]._serialized_start = 1913 + _globals["_DATATYPE_SHORT"]._serialized_end = 1978 + _globals["_DATATYPE_INTEGER"]._serialized_start = 1980 + _globals["_DATATYPE_INTEGER"]._serialized_end = 2047 + _globals["_DATATYPE_LONG"]._serialized_start = 2049 + _globals["_DATATYPE_LONG"]._serialized_end = 2113 + _globals["_DATATYPE_FLOAT"]._serialized_start = 2115 + _globals["_DATATYPE_FLOAT"]._serialized_end = 2180 + _globals["_DATATYPE_DOUBLE"]._serialized_start = 2182 + _globals["_DATATYPE_DOUBLE"]._serialized_end = 2248 + _globals["_DATATYPE_STRING"]._serialized_start = 2250 + _globals["_DATATYPE_STRING"]._serialized_end = 2346 + _globals["_DATATYPE_BINARY"]._serialized_start = 2348 + _globals["_DATATYPE_BINARY"]._serialized_end = 2414 + _globals["_DATATYPE_NULL"]._serialized_start = 2416 + _globals["_DATATYPE_NULL"]._serialized_end = 2480 + _globals["_DATATYPE_TIMESTAMP"]._serialized_start = 2482 + _globals["_DATATYPE_TIMESTAMP"]._serialized_end = 2551 + _globals["_DATATYPE_DATE"]._serialized_start = 2553 + _globals["_DATATYPE_DATE"]._serialized_end = 2617 + _globals["_DATATYPE_TIMESTAMPNTZ"]._serialized_start = 2619 + _globals["_DATATYPE_TIMESTAMPNTZ"]._serialized_end = 2691 + _globals["_DATATYPE_TIME"]._serialized_start = 2693 + _globals["_DATATYPE_TIME"]._serialized_end = 2806 + _globals["_DATATYPE_CALENDARINTERVAL"]._serialized_start = 2808 + _globals["_DATATYPE_CALENDARINTERVAL"]._serialized_end = 2884 + _globals["_DATATYPE_YEARMONTHINTERVAL"]._serialized_start = 2887 + _globals["_DATATYPE_YEARMONTHINTERVAL"]._serialized_end = 3066 + _globals["_DATATYPE_DAYTIMEINTERVAL"]._serialized_start = 3069 + _globals["_DATATYPE_DAYTIMEINTERVAL"]._serialized_end = 3246 + _globals["_DATATYPE_CHAR"]._serialized_start = 3248 + _globals["_DATATYPE_CHAR"]._serialized_end = 3336 + _globals["_DATATYPE_VARCHAR"]._serialized_start = 3338 + _globals["_DATATYPE_VARCHAR"]._serialized_end = 3429 + _globals["_DATATYPE_DECIMAL"]._serialized_start = 3432 + _globals["_DATATYPE_DECIMAL"]._serialized_end = 3585 + _globals["_DATATYPE_STRUCTFIELD"]._serialized_start = 3588 + _globals["_DATATYPE_STRUCTFIELD"]._serialized_end = 3749 + _globals["_DATATYPE_STRUCT"]._serialized_start = 3751 + _globals["_DATATYPE_STRUCT"]._serialized_end = 3878 + _globals["_DATATYPE_ARRAY"]._serialized_start = 3881 + _globals["_DATATYPE_ARRAY"]._serialized_end = 4043 + _globals["_DATATYPE_MAP"]._serialized_start = 4046 + _globals["_DATATYPE_MAP"]._serialized_end = 4265 + _globals["_DATATYPE_GEOMETRY"]._serialized_start = 4267 + _globals["_DATATYPE_GEOMETRY"]._serialized_end = 4355 + _globals["_DATATYPE_GEOGRAPHY"]._serialized_start = 4357 + _globals["_DATATYPE_GEOGRAPHY"]._serialized_end = 4446 + _globals["_DATATYPE_VARIANT"]._serialized_start = 4448 + _globals["_DATATYPE_VARIANT"]._serialized_end = 4515 + _globals["_DATATYPE_UDT"]._serialized_start = 4518 + _globals["_DATATYPE_UDT"]._serialized_end = 4807 + _globals["_DATATYPE_UNPARSED"]._serialized_start = 4809 + _globals["_DATATYPE_UNPARSED"]._serialized_end = 4861 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/types_pb2.pyi b/python/pyspark/sql/connect/proto/types_pb2.pyi index d46770c4f888..3f625890a809 100644 --- a/python/pyspark/sql/connect/proto/types_pb2.pyi +++ b/python/pyspark/sql/connect/proto/types_pb2.pyi @@ -674,6 +674,46 @@ class DataType(google.protobuf.message.Message): ], ) -> None: ... + class Geometry(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SRID_FIELD_NUMBER: builtins.int + TYPE_VARIATION_REFERENCE_FIELD_NUMBER: builtins.int + srid: builtins.int + type_variation_reference: builtins.int + def __init__( + self, + *, + srid: builtins.int = ..., + type_variation_reference: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "srid", b"srid", "type_variation_reference", b"type_variation_reference" + ], + ) -> None: ... + + class Geography(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + SRID_FIELD_NUMBER: builtins.int + TYPE_VARIATION_REFERENCE_FIELD_NUMBER: builtins.int + srid: builtins.int + type_variation_reference: builtins.int + def __init__( + self, + *, + srid: builtins.int = ..., + type_variation_reference: builtins.int = ..., + ) -> None: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "srid", b"srid", "type_variation_reference", b"type_variation_reference" + ], + ) -> None: ... + class Variant(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -821,6 +861,8 @@ class DataType(google.protobuf.message.Message): MAP_FIELD_NUMBER: builtins.int VARIANT_FIELD_NUMBER: builtins.int UDT_FIELD_NUMBER: builtins.int + GEOMETRY_FIELD_NUMBER: builtins.int + GEOGRAPHY_FIELD_NUMBER: builtins.int UNPARSED_FIELD_NUMBER: builtins.int TIME_FIELD_NUMBER: builtins.int @property @@ -878,6 +920,11 @@ class DataType(google.protobuf.message.Message): def udt(self) -> global___DataType.UDT: """UserDefinedType""" @property + def geometry(self) -> global___DataType.Geometry: + """Geospatial types""" + @property + def geography(self) -> global___DataType.Geography: ... + @property def unparsed(self) -> global___DataType.Unparsed: """UnparsedDataType""" @property @@ -909,6 +956,8 @@ class DataType(google.protobuf.message.Message): map: global___DataType.Map | None = ..., variant: global___DataType.Variant | None = ..., udt: global___DataType.UDT | None = ..., + geometry: global___DataType.Geometry | None = ..., + geography: global___DataType.Geography | None = ..., unparsed: global___DataType.Unparsed | None = ..., time: global___DataType.Time | None = ..., ) -> None: ... @@ -937,6 +986,10 @@ class DataType(google.protobuf.message.Message): b"double", "float", b"float", + "geography", + b"geography", + "geometry", + b"geometry", "integer", b"integer", "kind", @@ -996,6 +1049,10 @@ class DataType(google.protobuf.message.Message): b"double", "float", b"float", + "geography", + b"geography", + "geometry", + b"geometry", "integer", b"integer", "kind", @@ -1058,6 +1115,8 @@ class DataType(google.protobuf.message.Message): "map", "variant", "udt", + "geometry", + "geography", "unparsed", "time", ] diff --git a/sql/connect/common/src/main/protobuf/spark/connect/types.proto b/sql/connect/common/src/main/protobuf/spark/connect/types.proto index 1800e3885774..caaa2340f95d 100644 --- a/sql/connect/common/src/main/protobuf/spark/connect/types.proto +++ b/sql/connect/common/src/main/protobuf/spark/connect/types.proto @@ -67,15 +67,17 @@ message DataType { // UserDefinedType UDT udt = 23; + // Geospatial types + Geometry geometry = 26; + + Geography geography = 27; + // UnparsedDataType Unparsed unparsed = 24; Time time = 28; } - // Reserved for geometry and geography types - reserved 26, 27; - message Boolean { uint32 type_variation_reference = 1; } @@ -192,6 +194,16 @@ message DataType { uint32 type_variation_reference = 4; } + message Geometry { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + + message Geography { + int32 srid = 1; + uint32 type_variation_reference = 2; + } + message Variant { uint32 type_variation_reference = 1; }