Initial commit

facebookincubator · Nov 11, 2023 · eaa0b75 · eaa0b75
1 parent ec0abed
commit eaa0b75
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 3 deletions.
diff --git a/velox/expression/CastExpr.cpp b/velox/expression/CastExpr.cpp
@@ -338,6 +338,16 @@ VectorPtr CastExpr::applyArray(
   return result;
 }
 
+// Cast from unkown type to other types.
+VectorPtr CastExpr::applyUnknown(
+    const SelectivityVector& rows,
+    const BaseVector& /*input*/,
+    exec::EvalCtx& context,
+    const TypePtr& /*fromType*/,
+    const TypePtr& toType) {
+  return BaseVector::createNullConstant(toType, rows.end(), context.pool());
+}
+
 VectorPtr CastExpr::applyRow(
     const SelectivityVector& rows,
     const RowVector* input,
@@ -608,6 +618,9 @@ void CastExpr::applyPeeled(
             fromType->asRow(),
             toType);
         break;
+      case TypeKind::UNKNOWN:
+        result = applyUnknown(rows, input, context, fromType, toType);
+        break;
       default: {
         // Handle primitive type conversions.
         VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH(

diff --git a/velox/expression/CastExpr.h b/velox/expression/CastExpr.h
@@ -106,6 +106,13 @@ class CastExpr : public SpecialForm {
       const TypePtr& toType,
       VectorPtr& result);
 
+  VectorPtr applyUnknown(
+      const SelectivityVector& rows,
+      const BaseVector& /*input*/,
+      exec::EvalCtx& context,
+      const TypePtr& /*fromType*/,
+      const TypePtr& toType);
+
   VectorPtr applyMap(
       const SelectivityVector& rows,
       const MapVector* input,

diff --git a/velox/type/Type.h b/velox/type/Type.h
@@ -129,6 +129,10 @@ struct UnknownValue {
   bool operator>=(const UnknownValue& /* b */) const {
     return true;
   }
+
+  operator std::string() const {
+    return "NULL";
+  }
 };
 
 template <typename T>
@@ -1420,6 +1424,10 @@ std::shared_ptr<const OpaqueType> OPAQUE() {
         return TEMPLATE_FUNC<::facebook::velox::TypeKind::TIMESTAMP>(         \
             __VA_ARGS__);                                                     \
       }                                                                       \
+      case ::facebook::velox::TypeKind::UNKNOWN: {                            \
+        return TEMPLATE_FUNC<::facebook::velox::TypeKind::UNKNOWN>(           \
+            __VA_ARGS__);                                                     \
+      }                                                                       \
       default:                                                                \
         VELOX_FAIL(                                                           \
             "not a scalar type! kind: {}", mapTypeKindToName(typeKind));      \
@@ -1552,8 +1560,15 @@ std::shared_ptr<const OpaqueType> OPAQUE() {
     }                                                                          \
   }()
 
-#define VELOX_DYNAMIC_TYPE_DISPATCH(TEMPLATE_FUNC, typeKind, ...) \
-  VELOX_DYNAMIC_TYPE_DISPATCH_IMPL(TEMPLATE_FUNC, , typeKind, __VA_ARGS__)
+#define VELOX_DYNAMIC_TYPE_DISPATCH(TEMPLATE_FUNC, typeKind, ...)              \
+  [&]() {                                                                      \
+    if ((typeKind) == ::facebook::velox::TypeKind::UNKNOWN) {                  \
+      return TEMPLATE_FUNC<::facebook::velox::TypeKind::UNKNOWN>(__VA_ARGS__); \
+    } else {                                                                   \
+      return VELOX_DYNAMIC_TYPE_DISPATCH_IMPL(                                 \
+          TEMPLATE_FUNC, , typeKind, __VA_ARGS__);                             \
+    }                                                                          \
+  }()
 
 #define VELOX_DYNAMIC_TYPE_DISPATCH_ALL(TEMPLATE_FUNC, typeKind, ...)          \
   [&]() {                                                                      \
@@ -2346,6 +2361,16 @@ struct IsRowType<Row<Ts...>> {
 
 } // namespace facebook::velox
 
+namespace std {
+template <>
+struct hash<::facebook::velox::UnknownValue> {
+  size_t operator()(const ::facebook::velox::UnknownValue& /* value */) const {
+    return 0;
+  }
+};
+
+} // namespace std
+
 namespace folly {
 template <>
 struct hasher<::facebook::velox::UnknownValue> {

diff --git a/velox/vector/arrow/Bridge.cpp b/velox/vector/arrow/Bridge.cpp
@@ -257,6 +257,8 @@ const char* exportArrowFormatStr(
       return "+m"; // map
     case TypeKind::ROW:
       return "+s"; // struct
+    case TypeKind::UNKNOWN:
+      return "n";
 
     default:
       VELOX_NYI("Unable to map type '{}' to ArrowSchema.", type->kind());
@@ -596,6 +598,7 @@ void exportFlat(
     case TypeKind::REAL:
     case TypeKind::DOUBLE:
     case TypeKind::TIMESTAMP:
+    case TypeKind::UNKNOWN:
       exportValues(vec, rows, out, pool, holder);
       break;
     case TypeKind::VARCHAR:
@@ -1024,6 +1027,8 @@ TypePtr importFromArrow(const ArrowSchema& arrowSchema) {
       return REAL();
     case 'g':
       return DOUBLE();
+    case 'n':
+      return UNKNOWN();
 
     // Map both utf-8 and large utf-8 string to varchar.
     case 'u':

diff --git a/velox/vector/arrow/tests/ArrowBridgeSchemaTest.cpp b/velox/vector/arrow/tests/ArrowBridgeSchemaTest.cpp
@@ -235,6 +235,7 @@ TEST_F(ArrowBridgeSchemaExportTest, constant) {
 }
 
 TEST_F(ArrowBridgeSchemaExportTest, unsupported) {
+  GTEST_SKIP() << "Skipping it, cause unknown type supported";
   // Try some combination of unsupported types to ensure there's no crash or
   // memory leak in failure scenarios.
   EXPECT_THROW(testScalarType(UNKNOWN(), ""), VeloxException);
@@ -349,7 +350,7 @@ TEST_F(ArrowBridgeSchemaImportTest, complexTypes) {
 }
 
 TEST_F(ArrowBridgeSchemaImportTest, unsupported) {
-  EXPECT_THROW(testSchemaImport("n"), VeloxUserError);
+  // EXPECT_THROW(testSchemaImport("n"), VeloxUserError);
   EXPECT_THROW(testSchemaImport("C"), VeloxUserError);
   EXPECT_THROW(testSchemaImport("S"), VeloxUserError);
   EXPECT_THROW(testSchemaImport("I"), VeloxUserError);