From df2be932797e8daf2e832f941c10e9761f2001d6 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 3 Mar 2022 10:11:28 +0800 Subject: [PATCH 1/3] 7.0.0 rebase gazelle test fix --- .../NativeSerializedRecordBatchIterator.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeSerializedRecordBatchIterator.java diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeSerializedRecordBatchIterator.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeSerializedRecordBatchIterator.java new file mode 100644 index 0000000000000..2e5a2706a2cb5 --- /dev/null +++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/NativeSerializedRecordBatchIterator.java @@ -0,0 +1,36 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.arrow.dataset.jni; + +import java.util.Iterator; + +/** + * Iterate on flatbuffers-serialized {@link org.apache.arrow.vector.ipc.message.ArrowRecordBatch}. + *

+ * {@link #next()} should be called from C++ scanner to read Java-generated Arrow data. + */ +public interface NativeSerializedRecordBatchIterator extends Iterator, AutoCloseable { + + /** + * Return next serialized {@link org.apache.arrow.vector.ipc.message.ArrowRecordBatch} Java + * byte array. + */ + @Override + byte[] next(); +} \ No newline at end of file From f5f742a010a2e5d1fac9abca9fe076db084afa0a Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 3 Mar 2022 16:20:59 +0800 Subject: [PATCH 2/3] fix --- cpp/src/jni/dataset/jni_wrapper.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/src/jni/dataset/jni_wrapper.cc b/cpp/src/jni/dataset/jni_wrapper.cc index 8ec8e859df8b7..407bfcb3b1803 100644 --- a/cpp/src/jni/dataset/jni_wrapper.cc +++ b/cpp/src/jni/dataset/jni_wrapper.cc @@ -84,7 +84,11 @@ class ReserveFromJava : public arrow::jniutil::ReservationListener { arrow::Status OnReservation(int64_t size) override { JNIEnv* env; if (vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { - return arrow::Status::Invalid("JNIEnv was not attached to current thread"); + if (vm_->AttachCurrentThreadAsDaemon( + reinterpret_cast(&env), nullptr) != JNI_OK) { + return arrow::Status::Invalid( + "JNIEnv was not able to be attached to current thread"); + } } env->CallObjectMethod(java_reservation_listener_, reserve_memory_method, size); RETURN_NOT_OK(arrow::jniutil::CheckException(env)); @@ -94,7 +98,11 @@ class ReserveFromJava : public arrow::jniutil::ReservationListener { arrow::Status OnRelease(int64_t size) override { JNIEnv* env; if (vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION) != JNI_OK) { - return arrow::Status::Invalid("JNIEnv was not attached to current thread"); + if (vm_->AttachCurrentThreadAsDaemon( + reinterpret_cast(&env), nullptr) != JNI_OK) { + return arrow::Status::Invalid( + "JNIEnv was not able to be attached to current thread"); + } } env->CallObjectMethod(java_reservation_listener_, unreserve_memory_method, size); RETURN_NOT_OK(arrow::jniutil::CheckException(env)); From 12917943d23055b0e7ca4cbc3484ae772b5d26af Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Thu, 3 Mar 2022 16:21:06 +0800 Subject: [PATCH 3/3] fix --- cpp/src/gandiva/function_registry_datetime.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cpp/src/gandiva/function_registry_datetime.cc b/cpp/src/gandiva/function_registry_datetime.cc index 4849c731b86f9..8415e594c8fdf 100644 --- a/cpp/src/gandiva/function_registry_datetime.cc +++ b/cpp/src/gandiva/function_registry_datetime.cc @@ -145,7 +145,8 @@ std::vector GetDateTimeFunctionRegistry() { DATE_TYPES(LAST_DAY_SAFE_NULL_IF_NULL, last_day, {}), BASE_NUMERIC_TYPES(TO_TIME_SAFE_NULL_IF_NULL, to_time, {}), - BASE_NUMERIC_TYPES(TO_TIMESTAMP_SAFE_NULL_IF_NULL, to_timestamp, {})}; + BASE_NUMERIC_TYPES(TO_TIMESTAMP_SAFE_NULL_IF_NULL, to_timestamp, {}), + NativeFunction("convertTimestampUnit", {}, DataTypeVector{timestamp()}, arrow::timestamp(arrow::TimeUnit::MICRO), kResultNullIfNull, "convertTimestampUnit_ms"), @@ -153,7 +154,7 @@ std::vector GetDateTimeFunctionRegistry() { kResultNullIfNull, "convertTimestampUnit_us"), NativeFunction("castDATE", {}, DataTypeVector{date64()}, date32(), - kResultNullIfNull, "castDATE_date64"), + kResultNullIfNull, "castDATE32_date64"), NativeFunction("castTIMESTAMP", {}, DataTypeVector{date32()}, timestamp(), kResultNullIfNull, "castTIMESTAMP_date32"), @@ -197,7 +198,8 @@ std::vector GetDateTimeFunctionRegistry() { NativeFunction("date_diff", {}, DataTypeVector{date32(), date32()}, int32(), kResultNullIfNull, "micros_to_timestamp_date32_date32"), - DATE_TYPES(LAST_DAY_SAFE_NULL_IF_NULL, last_day, {}); + DATE_TYPES(LAST_DAY_SAFE_NULL_IF_NULL, last_day, {}) + }; return date_time_fn_registry_; }