KUDU-1261 add GetArray/SetArray C++ client API

This patch adds methods into the Kudu client C++ API to set an array
of values for all the supported scalar types except for
INT128 and DECIMAL128.

Also, added corresponding KuduScanBatch::RowPtr::GetArrayXxx() methods.

Added src/kudu/client/array_cell.h with KuduArrayCellView API
to access raw data in array cells.  That's oriented to working with
raw data access API such as KuduScanBatch::RowPtr::cell() and
KuduScanBatch::direct_data(); AFAIK, that's what Impala uses to fetch
scan results from Kudu.

This patch is made to unblock the integration work dependent on the new
Kudu client API for array columns (i.e., integration with Impala).
So, it contains only partial test coverage for the newly added
functionality, primarily in partial_row-test.cc.  The end-to-end tests
for writing/reading array data are coming in a follow-up changelist.
They require server-side support of the array data in both MRS and
array data blocks persisted on the disk, and is a part of another
changelist [1].

Adding Doxygen documentation comments for the newly introduced methods
will be taken care in a separate changelist as well.

[1] https://gerrit.cloudera.org/#/c/22868/

Change-Id: Idba29337563cefefa8cf1000663ec7e27885cb6f
Reviewed-on: http://gerrit.cloudera.org:8080/23474
Tested-by: Alexey Serbin <alexey@apache.org>
Reviewed-by: Abhishek Chennaka <achennaka@cloudera.com>

Author: alexeyserbin

src/kudu/client/CMakeLists.txt

@@ -30,6 +30,7 @@ ADD_EXPORTABLE_LIBRARY(client_proto
   NONLINK_DEPS ${CLIENT_PROTO_TGTS})
 
 set(CLIENT_SRCS
+  array_cell-internal.cc
   authz_token_cache.cc
   batcher.cc
   client.cc
@@ -183,6 +184,7 @@ install(TARGETS kudu_client_exported
 
 # Headers: client
 install(FILES
+  array_cell.h
   callbacks.h
   client.h
   columnar_scan_batch.h

src/kudu/client/array_cell-internal.cc

@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#include <cstdint>
+#include <cstring>
+
+#include "kudu/client/array_cell.h"
+#include "kudu/client/schema-internal.h"
+#include "kudu/client/schema.h"
+#include "kudu/common/array_cell_view.h"
+#include "kudu/util/slice.h"
+#include "kudu/util/status.h"
+
+namespace kudu {
+namespace client {
+
+class KuduArrayCellView::Data {
+ public:
+  Data(const uint8_t* buf, const size_t size)
+      : view_(buf, size) {
+  }
+
+  ~Data() = default;
+
+  ArrayCellMetadataView view_;
+};
+
+KuduArrayCellView::KuduArrayCellView(const void* cell_ptr) {
+  if (!cell_ptr) {
+    data_ = new Data(nullptr, 0);
+  } else {
+    const Slice* cell = reinterpret_cast<const Slice*>(cell_ptr);
+    data_ = new Data(cell->data(), cell->size());
+  }
+}
+
+KuduArrayCellView::KuduArrayCellView(const uint8_t* buf, const size_t size)
+    : data_(new Data(buf, size)) {
+}
+
+KuduArrayCellView::~KuduArrayCellView() {
+  delete data_;
+}
+
+Status KuduArrayCellView::Init() {
+  return data_->view_.Init();
+}
+
+size_t KuduArrayCellView::elem_num() const {
+  return data_->view_.elem_num();
+}
+
+bool KuduArrayCellView::empty() const {
+  return data_->view_.empty();
+}
+
+const uint8_t* KuduArrayCellView::not_null_bitmap() const {
+  return data_->view_.not_null_bitmap();
+}
+
+const void* KuduArrayCellView::data(KuduColumnSchema::DataType data_type,
+                                    const KuduColumnTypeAttributes& attributes) const {
+  return data_->view_.data_as(ToInternalDataType(data_type, attributes));
+}
+
+} // namespace client
+} // namespace kudu

src/kudu/client/array_cell.h

@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef KUDU_CLIENT_ARRAY_CELL_H
+#define KUDU_CLIENT_ARRAY_CELL_H
+
+#ifdef KUDU_HEADERS_NO_STUBS
+#include "kudu/gutil/port.h"
+#else
+#include "kudu/client/stubs.h"
+#endif
+
+#include "kudu/client/schema.h"
+#include "kudu/util/kudu_export.h"
+#include "kudu/util/status.h"
+
+namespace kudu {
+namespace client {
+
+// TODO(aserbin): add doxygen comments
+class KUDU_EXPORT KuduArrayCellView {
+ public:
+  // cell_ptr: pointer to an array cell in scan results, returned by, e.g.,
+  // by KuduScanBatch::RowPtr::cell(idx)
+  explicit KuduArrayCellView(const void* cell_ptr);
+  // buf: data raw pointer
+  // len: size of the buffer (bytes) pointed at by the 'buf' pointer
+  KuduArrayCellView(const uint8_t* buf, size_t size);
+  ~KuduArrayCellView();
+
+  // Process the input data. This method must be called once prior to calling
+  // any other methods of this class.
+  Status Init();
+
+  // Number of elements in the array.
+  size_t elem_num() const;
+
+  // Whether the array cell is empty, i.e. does not contain any elements.
+  bool empty() const;
+
+  // Get non-null (a.k.a. validity) bitmap for the array elements.
+  const uint8_t* not_null_bitmap() const;
+
+  // Accessor for the cell's raw data in the format similar to what
+  // KuduScanBatch::RowPtr::direct_data() provides
+  const void* data(
+      KuduColumnSchema::DataType data_type,
+      const KuduColumnTypeAttributes& attributes = KuduColumnTypeAttributes()) const;
+
+ private:
+  class KUDU_NO_EXPORT Data;
+
+  // Owned.
+  Data* data_;
+
+  DISALLOW_COPY_AND_ASSIGN(KuduArrayCellView);
+};
+
+} // namespace client
+} // namespace kudu
+
+#endif // #ifndef KUDU_CLIENT_ARRAY_CELL_H ...

src/kudu/client/scan_batch.cc

@@ -17,13 +17,17 @@
 
 #include "kudu/client/scan_batch.h"
 
+#include <algorithm>
+#include <iterator>
 #include <cstring>
 #include <string>
+#include <vector>
 
 #include <glog/logging.h>
 
 #include "kudu/client/row_result.h"
 #include "kudu/client/scanner-internal.h"
+#include "kudu/common/array_cell_view.h"
 #include "kudu/common/common.pb.h"
 #include "kudu/common/schema.h"
 #include "kudu/common/types.h"
@@ -33,6 +37,7 @@
 #include "kudu/util/logging.h"
 
 using std::string;
+using std::vector;
 using strings::Substitute;
 
 namespace kudu {
@@ -255,6 +260,173 @@ Status KuduScanBatch::RowPtr::Get(int col_idx, typename T::cpp_type* val) const
   return Status::OK();
 }
 
+namespace {
+
+Status ArrayValidation(const ColumnSchema& col,
+                       const char* type_name) {
+  if (PREDICT_FALSE(col.type_info()->type() != NESTED)) {
+    return BadTypeStatus(type_name, col);
+  }
+  const auto* descriptor = col.type_info()->nested_type_info();
+  if (PREDICT_FALSE(!descriptor)) {
+    return Status::InvalidArgument(Substitute(
+        "column '$0': missing type descriptor for NESTED type", col.name()));
+  }
+  if (PREDICT_FALSE(!descriptor->is_array())) {
+    return Status::InvalidArgument(Substitute(
+        "column '$0': underlying NESTED type isn't an array", col.name()));
+  }
+  return Status::OK();
+}
+
+} // anonymous namespace
+
+template<typename T>
+Status KuduScanBatch::RowPtr::GetArray(const Slice& col_name,
+                                       vector<typename T::cpp_type>* data_out,
+                                       vector<bool>* validity_out) const {
+  int col_idx;
+  RETURN_NOT_OK(schema_->FindColumn(col_name, &col_idx));
+  return GetArray<T>(col_idx, data_out, validity_out);
+}
+
+template<typename T>
+Status KuduScanBatch::RowPtr::GetArray(int col_idx,
+                                       vector<typename T::cpp_type>* data_out,
+                                       vector<bool>* validity_out) const {
+  const ColumnSchema& col = schema_->column(col_idx);
+  RETURN_NOT_OK(ArrayValidation(col, T::name()));
+  if (PREDICT_FALSE(col.is_nullable() && IsNull(col_idx))) {
+    return Status::NotFound("column is NULL");
+  }
+  const Slice* cell_data = reinterpret_cast<const Slice*>(
+      row_data_ + schema_->column_offset(col_idx));
+  ArrayCellMetadataView view(cell_data->data(), cell_data->size());
+  RETURN_NOT_OK(view.Init());
+
+  if (data_out) {
+    data_out->resize(view.elem_num());
+    if (!view.empty()) {
+      const uint8_t* data_raw = view.data_as(T::type);
+      DCHECK(data_raw);
+      memcpy(data_out->data(), data_raw, view.elem_num() * sizeof(typename T::cpp_type));
+    }
+  }
+  if (validity_out) {
+    validity_out->resize(view.elem_num());
+    if (!view.empty()) {
+      *validity_out = BitmapToVector(view.not_null_bitmap(), view.elem_num());
+    }
+  }
+  return Status::OK();
+}
+
+// Since std::vector<bool> isn't a standard container, the data() accessor
+// isn't available and copying the data requires an alternative approach.
+template<>
+Status KuduScanBatch::RowPtr::GetArray<TypeTraits<BOOL>>(
+    int col_idx,
+    vector<bool>* data_out,
+    vector<bool>* validity) const {
+  const ColumnSchema& col = schema_->column(col_idx);
+  RETURN_NOT_OK(ArrayValidation(col, TypeTraits<BOOL>::name()));
+  if (PREDICT_FALSE(col.is_nullable() && IsNull(col_idx))) {
+    return Status::NotFound("column is NULL");
+  }
+  const Slice* cell_data = reinterpret_cast<const Slice*>(
+      row_data_ + schema_->column_offset(col_idx));
+  ArrayCellMetadataView view(cell_data->data(), cell_data->size());
+  RETURN_NOT_OK(view.Init());
+
+  if (data_out) {
+    const size_t elem_num = view.elem_num();
+    data_out->clear();
+    data_out->reserve(elem_num);
+    const uint8_t* data_raw = view.data_as(BOOL);
+    DCHECK(data_raw);
+    std::copy(data_raw, data_raw + elem_num, std::back_inserter(*data_out));
+  }
+  if (validity) {
+    *validity = BitmapToVector(view.not_null_bitmap(), view.elem_num());
+  }
+  return Status::OK();
+}
+
+Status KuduScanBatch::RowPtr::GetArrayBool(int col_idx,
+                                           vector<bool>* data,
+                                           vector<bool>* validity) const {
+  return GetArray<TypeTraits<BOOL>>(col_idx, data, validity);
+}
+
+Status KuduScanBatch::RowPtr::GetArrayInt8(int col_idx,
+                                           vector<int8_t>* data,
+                                           vector<bool>* validity) const {
+  return GetArray<TypeTraits<INT8>>(col_idx, data, validity);
+}
+
+Status KuduScanBatch::RowPtr::GetArrayInt16(int col_idx,
+                                            vector<int16_t>* data,
+                                            vector<bool>* validity) const {
+  return GetArray<TypeTraits<INT16>>(col_idx, data, validity);
+}
+
+Status KuduScanBatch::RowPtr::GetArrayInt32(int col_idx,
+                                            vector<int32_t>* data,
+                                            vector<bool>* validity) const {
+  return GetArray<TypeTraits<INT32>>(col_idx, data, validity);
+}
+
+Status KuduScanBatch::RowPtr::GetArrayInt64(int col_idx,
+                                            vector<int64_t>* data,
+                                            vector<bool>* validity) const {
+  return GetArray<TypeTraits<INT64>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayUnscaledDecimal(int col_idx,
+                                                      vector<int32_t>* data,
+                                                      vector<bool>* validity) const {
+  return GetArray<TypeTraits<DECIMAL32>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayUnscaledDecimal(int col_idx,
+                                                      vector<int64_t>* data,
+                                                      vector<bool>* validity) const {
+  return GetArray<TypeTraits<DECIMAL64>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayUnixTimeMicros(int col_idx,
+                                                     vector<int64_t>* data,
+                                                     vector<bool>* validity) const {
+  return GetArray<TypeTraits<UNIXTIME_MICROS>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayDate(int col_idx,
+                                           vector<int32_t>* data,
+                                           vector<bool>* validity) const {
+  return GetArray<TypeTraits<DATE>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayFloat(int col_idx,
+                                            vector<float>* data,
+                                            vector<bool>* validity) const {
+  return GetArray<TypeTraits<FLOAT>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayDouble(int col_idx,
+                                             vector<double>* data,
+                                             vector<bool>* validity) const {
+  return GetArray<TypeTraits<DOUBLE>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayString(int col_idx,
+                                             vector<Slice>* data,
+                                             vector<bool>* validity) const {
+  return GetArray<TypeTraits<STRING>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayBinary(int col_idx,
+                                            vector<Slice>* data,
+                                            vector<bool>* validity) const {
+  return GetArray<TypeTraits<BINARY>>(col_idx, data, validity);
+}
+Status KuduScanBatch::RowPtr::GetArrayVarchar(int col_idx,
+                                              vector<Slice>* data,
+                                              vector<bool>* validity) const {
+  return GetArray<TypeTraits<VARCHAR>>(col_idx, data, validity);
+}
+
 const void* KuduScanBatch::RowPtr::cell(int col_idx) const {
   return row_data_ + schema_->column_offset(col_idx);
 }