Merge branch 'master' into sysinfo

tikv · Sep 1, 2022 · 7e3cbbb · 7e3cbbb
2 parents 59ede39 + 90c4a06
commit 7e3cbbb
Show file tree

Hide file tree

Showing 12 changed files with 225 additions and 22 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/components/tidb_query_datatype/Cargo.toml b/components/tidb_query_datatype/Cargo.toml
@@ -6,6 +6,7 @@ publish = false
 description = "Data type of a query engine to run TiDB pushed down executors"
 
 [dependencies]
+base64 = "0.13"
 bitfield = "0.13.2"
 bitflags = "1.0.1"
 boolinator = "2.4.0"

diff --git a/components/tidb_query_datatype/src/codec/convert.rs b/components/tidb_query_datatype/src/codec/convert.rs
@@ -510,7 +510,7 @@ impl<'a> ToInt for JsonRef<'a> {
         // TiDB:  5
         // MySQL: 4
         let val = match self.get_type() {
-            JsonType::Object | JsonType::Array => Ok(ctx
+            JsonType::Object | JsonType::Array | JsonType::Opaque => Ok(ctx
                 .handle_truncate_err(Error::truncated_wrong_val("Integer", self.to_string()))
                 .map(|_| 0)?),
             JsonType::Literal => Ok(self.get_literal().map_or(0, |x| x as i64)),
@@ -526,7 +526,7 @@ impl<'a> ToInt for JsonRef<'a> {
     #[inline]
     fn to_uint(&self, ctx: &mut EvalContext, tp: FieldTypeTp) -> Result<u64> {
         let val = match self.get_type() {
-            JsonType::Object | JsonType::Array => Ok(ctx
+            JsonType::Object | JsonType::Array | JsonType::Opaque => Ok(ctx
                 .handle_truncate_err(Error::truncated_wrong_val("Integer", self.to_string()))
                 .map(|_| 0)?),
             JsonType::Literal => Ok(self.get_literal().map_or(0, |x| x as u64)),

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/binary.rs b/components/tidb_query_datatype/src/codec/mysql/json/binary.rs
@@ -80,6 +80,14 @@ impl<'a> JsonRef<'a> {
                     &self.value()[val_offset..val_offset + str_len as usize + len_len],
                 )
             }
+            JsonType::Opaque => {
+                let (opaque_bytes_len, len_len) =
+                    NumberCodec::try_decode_var_u64(&self.value()[val_offset + 1..])?;
+                JsonRef::new(
+                    val_type,
+                    &self.value()[val_offset..val_offset + opaque_bytes_len as usize + len_len + 1],
+                )
+            }
             _ => {
                 let data_size =
                     NumberCodec::decode_u32_le(&self.value()[val_offset + ELEMENT_COUNT_LEN..])

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs b/components/tidb_query_datatype/src/codec/mysql/json/comparison.rs
@@ -37,6 +37,7 @@ impl<'a> JsonRef<'a> {
                 .map_or(PRECEDENCE_NULL, |_| PRECEDENCE_BOOLEAN),
             JsonType::I64 | JsonType::U64 | JsonType::Double => PRECEDENCE_NUMBER,
             JsonType::String => PRECEDENCE_STRING,
+            JsonType::Opaque => PRECEDENCE_OPAQUE,
         }
     }
 
@@ -140,6 +141,15 @@ impl<'a> PartialOrd for JsonRef<'a> {
                     }
                     Some(left_count.cmp(&right_count))
                 }
+                JsonType::Opaque => {
+                    if let (Ok(left), Ok(right)) =
+                        (self.get_opaque_bytes(), right.get_opaque_bytes())
+                    {
+                        left.partial_cmp(right)
+                    } else {
+                        return None;
+                    }
+                }
             };
         }
 

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs b/components/tidb_query_datatype/src/codec/mysql/json/jcodec.rs
@@ -5,7 +5,10 @@ use std::{collections::BTreeMap, convert::TryInto, f64, str};
 use codec::{number::NumberCodec, prelude::*};
 
 use super::{constants::*, Json, JsonRef, JsonType};
-use crate::codec::{Error, Result};
+use crate::{
+    codec::{Error, Result},
+    FieldTypeTp,
+};
 
 impl<'a> JsonRef<'a> {
     fn encoded_len(&self) -> usize {
@@ -211,6 +214,14 @@ pub trait JsonEncoder: NumberEncoder {
         self.write_bytes(bytes)?;
         Ok(())
     }
+
+    fn write_json_opaque(&mut self, typ: FieldTypeTp, bytes: &[u8]) -> Result<()> {
+        self.write_u8(typ.to_u8().unwrap())?;
+        let bytes_len = bytes.len() as u64;
+        self.write_var_u64(bytes_len)?;
+        self.write_bytes(bytes)?;
+        Ok(())
+    }
 }
 
 pub trait JsonDatumPayloadChunkEncoder: BufferWriter {
@@ -243,6 +254,12 @@ pub trait JsonDecoder: NumberDecoder {
             }
             JsonType::I64 | JsonType::U64 | JsonType::Double => self.read_bytes(NUMBER_LEN)?,
             JsonType::Literal => self.read_bytes(LITERAL_LEN)?,
+            JsonType::Opaque => {
+                let value = self.bytes();
+                // the first byte of opaque stores the MySQL type code
+                let (opaque_bytes_len, len_len) = NumberCodec::try_decode_var_u64(&value[1..])?;
+                self.read_bytes(opaque_bytes_len as usize + len_len + 1)?
+            }
         };
         Ok(Json::new(tp, Vec::from(value)))
     }

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/json_type.rs b/components/tidb_query_datatype/src/codec/mysql/json/json_type.rs
@@ -1,6 +1,7 @@
 // Copyright 2017 TiKV Project Authors. Licensed under Apache-2.0.
 
 use super::{JsonRef, JsonType};
+use crate::FieldTypeTp;
 
 const JSON_TYPE_BOOLEAN: &[u8] = b"BOOLEAN";
 const JSON_TYPE_NONE: &[u8] = b"NULL";
@@ -10,6 +11,9 @@ const JSON_TYPE_DOUBLE: &[u8] = b"DOUBLE";
 const JSON_TYPE_STRING: &[u8] = b"STRING";
 const JSON_TYPE_OBJECT: &[u8] = b"OBJECT";
 const JSON_TYPE_ARRAY: &[u8] = b"ARRAY";
+const JSON_TYPE_BIT: &[u8] = b"BIT";
+const JSON_TYPE_BLOB: &[u8] = b"BLOB";
+const JSON_TYPE_OPAQUE: &[u8] = b"OPAQUE";
 
 impl<'a> JsonRef<'a> {
     /// `json_type` is the implementation for
@@ -26,6 +30,19 @@ impl<'a> JsonRef<'a> {
                 Some(_) => JSON_TYPE_BOOLEAN,
                 None => JSON_TYPE_NONE,
             },
+            JsonType::Opaque => match self.get_opaque_type() {
+                Ok(
+                    FieldTypeTp::TinyBlob
+                    | FieldTypeTp::MediumBlob
+                    | FieldTypeTp::LongBlob
+                    | FieldTypeTp::Blob
+                    | FieldTypeTp::String
+                    | FieldTypeTp::VarString
+                    | FieldTypeTp::VarChar,
+                ) => JSON_TYPE_BLOB,
+                Ok(FieldTypeTp::Bit) => JSON_TYPE_BIT,
+                _ => JSON_TYPE_OPAQUE,
+            },
         }
     }
 }

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/mod.rs b/components/tidb_query_datatype/src/codec/mysql/json/mod.rs
@@ -90,11 +90,12 @@ use super::super::{datum::Datum, Error, Result};
 use crate::{
     codec::{
         convert::ConvertTo,
-        data_type::{Decimal, Real},
+        data_type::{BytesRef, Decimal, Real},
         mysql,
         mysql::{Duration, Time, TimeType},
     },
     expr::EvalContext,
+    FieldTypeTp,
 };
 
 const ERR_CONVERT_FAILED: &str = "Can not covert from ";
@@ -109,6 +110,10 @@ pub enum JsonType {
     U64 = 0x0a,
     Double = 0x0b,
     String = 0x0c,
+
+    // It's a special value for the compatibility with MySQL.
+    // It will store the raw buffer containing unexpected type (e.g. Binary).
+    Opaque = 0x0d,
 }
 
 impl TryFrom<u8> for JsonType {
@@ -206,6 +211,20 @@ impl<'a> JsonRef<'a> {
         Ok(str::from_utf8(self.get_str_bytes()?)?)
     }
 
+    // Returns the opaque value in bytes
+    pub(crate) fn get_opaque_bytes(&self) -> Result<&'a [u8]> {
+        assert_eq!(self.type_code, JsonType::Opaque);
+        let val = self.value();
+        let (str_len, len_len) = NumberCodec::try_decode_var_u64(&val[1..])?;
+        Ok(&val[(len_len + 1)..len_len + 1 + str_len as usize])
+    }
+
+    pub(crate) fn get_opaque_type(&self) -> Result<FieldTypeTp> {
+        assert_eq!(self.type_code, JsonType::Opaque);
+        let val = self.value();
+        FieldTypeTp::from_u8(val[0]).ok_or(box_err!("invalid opaque type code"))
+    }
+
     // Return whether the value is zero.
     // https://dev.mysql.com/doc/refman/8.0/en/json.html#Converting%20between%20JSON%20and%20non-JSON%20values
     pub(crate) fn is_zero(&self) -> bool {
@@ -217,6 +236,7 @@ impl<'a> JsonRef<'a> {
             JsonType::U64 => self.get_u64() == 0,
             JsonType::Double => self.get_double() == 0f64,
             JsonType::String => false,
+            JsonType::Opaque => false,
         }
     }
 
@@ -284,6 +304,12 @@ impl Json {
         Ok(Self::new(JsonType::String, value))
     }
 
+    pub fn from_opaque(typ: FieldTypeTp, bytes: BytesRef<'_>) -> Result<Self> {
+        let mut value = vec![];
+        value.write_json_opaque(typ, bytes)?;
+        Ok(Self::new(JsonType::Opaque, value))
+    }
+
     /// Creates a `literal` JSON from a `bool`
     pub fn from_bool(b: bool) -> Result<Self> {
         let mut value = vec![];
@@ -414,7 +440,7 @@ impl<'a> ConvertTo<f64> for JsonRef<'a> {
     #[inline]
     fn convert(&self, ctx: &mut EvalContext) -> Result<f64> {
         let d = match self.get_type() {
-            JsonType::Array | JsonType::Object => ctx
+            JsonType::Array | JsonType::Object | JsonType::Opaque => ctx
                 .handle_truncate_err(Error::truncated_wrong_val("Float", self.to_string()))
                 .map(|_| 0f64)?,
             JsonType::U64 => self.get_u64() as f64,

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs b/components/tidb_query_datatype/src/codec/mysql/json/modifier.rs
@@ -232,7 +232,8 @@ impl<'a> BinaryModifier<'a> {
             | JsonType::I64
             | JsonType::U64
             | JsonType::Double
-            | JsonType::String => {
+            | JsonType::String
+            | JsonType::Opaque => {
                 buf.extend_from_slice(self.old.value);
             }
             JsonType::Object | JsonType::Array => {

diff --git a/components/tidb_query_datatype/src/codec/mysql/json/serde.rs b/components/tidb_query_datatype/src/codec/mysql/json/serde.rs
@@ -107,6 +107,17 @@ impl<'a> Serialize for JsonRef<'a> {
                 }
                 tup.end()
             }
+            JsonType::Opaque => {
+                let bytes = self
+                    .get_opaque_bytes()
+                    .map_err(|_| SerError::custom("invalid opaque value"))?;
+                let typ = self
+                    .get_opaque_type()
+                    .map_err(|_| SerError::custom("invalid opaque type code"))?;
+
+                let str = format!("base64:type{}:{}", typ, base64::encode(bytes));
+                serializer.serialize_str(&str)
+            }
         }
     }
 }

diff --git a/components/tidb_query_datatype/src/codec/row/v2/encoder_for_test.rs b/components/tidb_query_datatype/src/codec/row/v2/encoder_for_test.rs
@@ -6,7 +6,7 @@
 //! According to <https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md>
 //!
 //! The row format is:
-//! ```
+//! ```ignore
 //! | version | flag | number_of_non_null_columns | number_of_null_columns | non_null_column_ids | null_column_ids | value_offsets | values |
 //! |---------| ---- | -------------------------- | ---------------------- | ------------------- | --------------- | ------------- | ------ |
 //! ```