fix

apache · May 15, 2023 · 44187b1 · 44187b1
1 parent 637f46d
commit 44187b1
Show file tree

Hide file tree

Showing 13 changed files with 876 additions and 727 deletions.
diff --git a/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala b/backends-velox/src/main/scala/io/glutenproject/backendsapi/velox/VeloxBackend.scala
@@ -70,7 +70,9 @@ object VeloxBackendSettings extends BackendSettings {
     format match {
       case ParquetReadFormat => validateTypes && validateFilePath
       case DwrfReadFormat => true
-      case OrcReadFormat => true
+      case OrcReadFormat => fields.map(_.dataType).collect {
+        case _: TimestampType =>
+      }.isEmpty
       case _ => false
     }
   }

diff --git a/cpp/velox/compute/VeloxColumnarToRowConverter.cc b/cpp/velox/compute/VeloxColumnarToRowConverter.cc
@@ -185,10 +185,10 @@ arrow::Status VeloxColumnarToRowConverter::write() {
               setNullAt(bufferAddress_, offsets_[rowIdx], field_offset, col_idx);
             }
           } else {
-            auto longDecimal = vec->asFlatVector<velox::UnscaledLongDecimal>()->rawValues();
             if (flag) {
               setNullAt(bufferAddress_, offsets_[rowIdx], field_offset, col_idx);
             } else {
+              auto longDecimal = vec->asFlatVector<velox::UnscaledLongDecimal>()->rawValues();
               int32_t size;
               velox::int128_t veloxInt128 = longDecimal[rowIdx].unscaledValue();
 
@@ -212,7 +212,7 @@ arrow::Status VeloxColumnarToRowConverter::write() {
         }
         break;
       }
-      case arrow::Time64Type::type_id: {
+      case arrow::TimestampType::type_id: {
         SERIALIZE_COLUMN(TimestampType);
         break;
       }

diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -40,6 +40,40 @@ const std::string kTotalScanTime = "totalScanTime";
 // others
 const std::string kHiveDefaultPartition = "__HIVE_DEFAULT_PARTITION__";
 std::atomic<int32_t> taskSerial;
+
+// From_hex from dlib.
+inline unsigned char fromHex(unsigned char ch) {
+  if (ch <= '9' && ch >= '0')
+    ch -= '0';
+  else if (ch <= 'f' && ch >= 'a')
+    ch -= 'a' - 10;
+  else if (ch <= 'F' && ch >= 'A')
+    ch -= 'A' - 10;
+  else
+    ch = 0;
+  return ch;
+}
+
+// URL decoder from dlib.
+const std::string urlDecode(const std::string& str) {
+  std::string result;
+  std::string::size_type i;
+  for (i = 0; i < str.size(); ++i) {
+    if (str[i] == '+') {
+      result += ' ';
+    } else if (str[i] == '%' && str.size() > i + 2) {
+      const unsigned char ch1 = fromHex(str[i + 1]);
+      const unsigned char ch2 = fromHex(str[i + 2]);
+      const unsigned char ch = (ch1 << 4) | ch2;
+      result += ch;
+      i += 2;
+    } else {
+      result += str[i];
+    }
+  }
+  return result;
+}
+
 } // namespace
 
 std::shared_ptr<velox::core::QueryCtx> WholeStageResultIterator::createNewVeloxQueryCtx() {
@@ -353,8 +387,8 @@ WholeStageResultIteratorFirstStage::extractPartitionColumnAndValue(const std::st
     if (partitionValue == kHiveDefaultPartition) {
       partitionKeys[partitionColumn] = std::nullopt;
     } else {
-      // Set to the map of partition keys.
-      partitionKeys[partitionColumn] = partitionValue;
+      // Set to the map of partition keys. Timestamp could be URL encoded.
+      partitionKeys[partitionColumn] = urlDecode(partitionValue);
     }
     // For processing the remaining keys.
     str = latterPart.substr(pos + 1);