Bundle 13.0.0-1 (2023-08-29)

README.md

@@ -1,43 +1,43 @@
-# arrow 12.0.1-1
+# arrow 13.0.0-1
 
- - mingw-w64-i686-arrow-12.0.1-1-any.pkg.tar.xz
+ - mingw-w64-i686-arrow-13.0.0-1-any.pkg.tar.xz
  - mingw-w64-i686-aws-sdk-cpp-1.7.365-1-any.pkg.tar.xz
  - mingw-w64-i686-brotli-1.0.9-4-any.pkg.tar.xz
- - mingw-w64-i686-openssl-1.1.1.m-9800-any.pkg.tar.xz
+ - mingw-w64-i686-openssl-3.1.1-9800-any.pkg.tar.xz
  - mingw-w64-i686-lz4-1.8.2-1-any.pkg.tar.xz
  - mingw-w64-i686-re2-20200801-1-any.pkg.tar.xz
  - mingw-w64-i686-snappy-1.1.7-2-any.pkg.tar.xz
  - mingw-w64-i686-bzip2-1.0.8-1-any.pkg.tar.xz
- - mingw-w64-i686-curl-7.84.0-9000-any.pkg.tar.xz
- - mingw-w64-i686-libssh2-1.11.0-9800-any.pkg.tar.xz
+ - mingw-w64-i686-curl-8.1.2-9000-any.pkg.tar.xz
+ - mingw-w64-i686-libssh2-1.11.0-9801-any.pkg.tar.xz
  - mingw-w64-i686-thrift-0.13.0-1-any.pkg.tar.xz
  - mingw-w64-i686-zstd-1.4.4-1-any.pkg.tar.xz
  - mingw-w64-i686-libutf8proc-2.4.0-2-any.pkg.tar.xz
  - mingw-w64-i686-nghttp2-1.51.0-1-any.pkg.tar.xz
- - mingw-w64-x86_64-arrow-12.0.1-1-any.pkg.tar.xz
+ - mingw-w64-x86_64-arrow-13.0.0-1-any.pkg.tar.xz
  - mingw-w64-x86_64-aws-sdk-cpp-1.7.365-1-any.pkg.tar.xz
  - mingw-w64-x86_64-brotli-1.0.9-4-any.pkg.tar.xz
- - mingw-w64-x86_64-openssl-1.1.1.m-9800-any.pkg.tar.xz
+ - mingw-w64-x86_64-openssl-3.1.1-9800-any.pkg.tar.xz
  - mingw-w64-x86_64-lz4-1.8.2-1-any.pkg.tar.xz
  - mingw-w64-x86_64-re2-20200801-1-any.pkg.tar.xz
  - mingw-w64-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
  - mingw-w64-x86_64-bzip2-1.0.8-1-any.pkg.tar.xz
- - mingw-w64-x86_64-curl-7.84.0-9000-any.pkg.tar.xz
- - mingw-w64-x86_64-libssh2-1.11.0-9800-any.pkg.tar.xz
+ - mingw-w64-x86_64-curl-8.1.2-9000-any.pkg.tar.xz
+ - mingw-w64-x86_64-libssh2-1.11.0-9801-any.pkg.tar.xz
  - mingw-w64-x86_64-thrift-0.13.0-1-any.pkg.tar.xz
  - mingw-w64-x86_64-zstd-1.4.4-1-any.pkg.tar.xz
  - mingw-w64-x86_64-libutf8proc-2.4.0-2-any.pkg.tar.xz
  - mingw-w64-x86_64-nghttp2-1.51.0-1-any.pkg.tar.xz
- - mingw-w64-ucrt-x86_64-arrow-12.0.1-1-any.pkg.tar.xz
+ - mingw-w64-ucrt-x86_64-arrow-13.0.0-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-aws-sdk-cpp-1.7.365-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-brotli-1.0.9-4-any.pkg.tar.xz
- - mingw-w64-ucrt-x86_64-openssl-1.1.1.m-9800-any.pkg.tar.xz
+ - mingw-w64-ucrt-x86_64-openssl-3.1.1-9800-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-lz4-1.8.2-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-re2-20200801-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-bzip2-1.0.8-1-any.pkg.tar.xz
- - mingw-w64-ucrt-x86_64-curl-7.84.0-9000-any.pkg.tar.xz
- - mingw-w64-ucrt-x86_64-libssh2-1.11.0-9800-any.pkg.tar.xz
+ - mingw-w64-ucrt-x86_64-curl-8.1.2-9000-any.pkg.tar.xz
+ - mingw-w64-ucrt-x86_64-libssh2-1.11.0-9801-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-thrift-0.13.0-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-zstd-1.4.4-1-any.pkg.tar.xz
  - mingw-w64-ucrt-x86_64-libutf8proc-2.4.0-2-any.pkg.tar.xz

include/arrow/acero/api.h

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// NOTE: API is EXPERIMENTAL and will change without going through a
+// deprecation cycle
+
+#pragma once
+
+/// \defgroup acero-api Utilities for creating and executing execution plans
+/// @{
+/// @}
+
+/// \defgroup acero-nodes Options classes for the various exec nodes
+/// @{
+/// @}
+
+#include "arrow/acero/exec_plan.h"
+#include "arrow/acero/options.h"

include/arrow/acero/asof_join_node.h

@@ -30,16 +30,11 @@ using AsofJoinKeys = AsofJoinNodeOptions::Keys;
 
 /// \brief Make the output schema of an as-of-join node
 ///
-/// Optionally, also provides the field output indices for this node.
-/// \see arrow::engine::RelationInfo
-///
 /// \param[in] input_schema the schema of each input to the node
 /// \param[in] input_keys the key of each input to the node
-/// \param[out] field_output_indices the output index of each field
 ARROW_ACERO_EXPORT Result<std::shared_ptr<Schema>> MakeOutputSchema(
     const std::vector<std::shared_ptr<Schema>>& input_schema,
-    const std::vector<AsofJoinKeys>& input_keys,
-    std::vector<int>* field_output_indices = NULLPTR);
+    const std::vector<AsofJoinKeys>& input_keys);
 
 }  // namespace asofjoin
 }  // namespace acero

include/arrow/acero/exec_plan.h

@@ -48,7 +48,7 @@ using compute::threaded_exec_context;
 
 namespace acero {
 
-/// \addtogroup execnode-components
+/// \addtogroup acero-internals
 /// @{
 
 class ARROW_ACERO_EXPORT ExecPlan : public std::enable_shared_from_this<ExecPlan> {
@@ -118,6 +118,10 @@ class ARROW_ACERO_EXPORT ExecPlan : public std::enable_shared_from_this<ExecPlan
   std::string ToString() const;
 };
 
+// Acero can be extended by providing custom implementations of ExecNode.  The methods
+// below are documented in detail and provide careful instruction on how to fulfill the
+// ExecNode contract.  It's suggested you familiarize yourself with the Acero
+// documentation in the C++ user guide.
 class ARROW_ACERO_EXPORT ExecNode {
  public:
   using NodeVector = std::vector<ExecNode*>;
@@ -173,9 +177,9 @@ class ARROW_ACERO_EXPORT ExecNode {
   /// non-deterministic.  For example, a hash-join has no predictable output order.
   ///
   /// If the ordering is Ordering::Implicit then there is a meaningful order but that
-  /// odering is not represented by any column in the data.  The most common case for this
-  /// is when reading data from an in-memory table.  The data has an implicit "row order"
-  /// which is not neccesarily represented in the data set.
+  /// ordering is not represented by any column in the data.  The most common case for
+  /// this is when reading data from an in-memory table.  The data has an implicit "row
+  /// order" which is not necessarily represented in the data set.
   ///
   /// A filter or project node will not modify the ordering.  Nothing needs to be done
   /// other than ensure the index assigned to output batches is the same as the
@@ -321,7 +325,7 @@ class ARROW_ACERO_EXPORT ExecNode {
   ///
   /// This is not a pause.  There will be no way to start the source again after this has
   /// been called.
-  Status StopProducing();
+  virtual Status StopProducing();
 
   std::string ToString(int indent = 0) const;
 
@@ -377,16 +381,36 @@ inline Result<ExecNode*> MakeExecNode(
   return factory(plan, std::move(inputs), options);
 }
 
-/// \brief Helper class for declaring sets of ExecNodes efficiently
+/// @}
+
+/// \addtogroup acero-api
+/// @{
+
+/// \brief Helper class for declaring execution nodes
 ///
-/// A Declaration represents an unconstructed ExecNode (and potentially more since its
-/// inputs may also be Declarations). The node can be constructed and added to a plan
-/// with Declaration::AddToPlan, which will recursively construct any inputs as necessary.
+/// A Declaration represents an unconstructed ExecNode (and potentially an entire graph
+/// since its inputs may also be Declarations)
+///
+/// A Declaration can be converted to a plan and executed using one of the
+/// DeclarationToXyz methods.
+///
+/// For more direct control, a Declaration can be added to an existing execution
+/// plan with Declaration::AddToPlan, which will recursively construct any inputs as
+/// necessary.
 struct ARROW_ACERO_EXPORT Declaration {
   using Input = std::variant<ExecNode*, Declaration>;
 
   Declaration() {}
 
+  /// \brief construct a declaration
+  /// \param factory_name the name of the exec node to construct.  The node must have
+  ///                     been added to the exec node registry with this name.
+  /// \param inputs the inputs to the node, these should be other declarations
+  /// \param options options that control the behavior of the node.  You must use
+  ///                the appropriate subclass.  For example, if `factory_name` is
+  ///                "project" then `options` should be ProjectNodeOptions.
+  /// \param label a label to give the node.  Can be used to distinguish it from other
+  ///              nodes of the same type in the plan.
   Declaration(std::string factory_name, std::vector<Input> inputs,
               std::shared_ptr<ExecNodeOptions> options, std::string label)
       : factory_name{std::move(factory_name)},
@@ -447,15 +471,28 @@ struct ARROW_ACERO_EXPORT Declaration {
   ///     });
   static Declaration Sequence(std::vector<Declaration> decls);
 
+  /// \brief add the declaration to an already created execution plan
+  /// \param plan the plan to add the node to
+  /// \param registry the registry to use to lookup the node factory
+  ///
+  /// This method will recursively call AddToPlan on all of the declaration's inputs.
+  /// This method is only for advanced use when the DeclarationToXyz methods are not
+  /// sufficient.
+  ///
+  /// \return the instantiated execution node
   Result<ExecNode*> AddToPlan(ExecPlan* plan, ExecFactoryRegistry* registry =
                                                   default_exec_factory_registry()) const;
 
   // Validate a declaration
   bool IsValid(ExecFactoryRegistry* registry = default_exec_factory_registry()) const;
 
+  /// \brief the name of the factory to use when creating a node
   std::string factory_name;
+  /// \brief the declarations's inputs
   std::vector<Input> inputs;
+  /// \brief options to control the behavior of the node
   std::shared_ptr<ExecNodeOptions> options;
+  /// \brief a label to give the node in the plan
   std::string label;
 };
 
@@ -489,7 +526,7 @@ struct ARROW_ACERO_EXPORT QueryOptions {
   /// otherwise.
   ///
   /// If explicitly set to true then plan execution will fail if there is no
-  /// meaningful ordering.  This can be useful to valdiate a query that should
+  /// meaningful ordering.  This can be useful to validate a query that should
   /// be emitting ordered results.
   ///
   /// If explicitly set to false then batches will be emit immediately even if there
@@ -513,6 +550,13 @@ struct ARROW_ACERO_EXPORT QueryOptions {
   /// the `use_threads` option.
   ::arrow::internal::Executor* custom_cpu_executor = NULLPTR;
 
+  /// \brief custom executor to use for IO work
+  ///
+  /// Must be null or remain valid for the duration of the plan.  If this is null then
+  /// the global io thread pool will be chosen whose behavior will be controlled by
+  /// the "ARROW_IO_THREADS" environment.
+  ::arrow::internal::Executor* custom_io_executor = NULLPTR;
+
   /// \brief a memory pool to use for allocations
   ///
   /// Must remain valid for the duration of the plan.
@@ -707,6 +751,10 @@ DeclarationToBatchesAsync(Declaration declaration, ExecContext exec_context);
 /// fills up.
 ///
 /// If a custom exec context is provided then the value of `use_threads` will be ignored.
+///
+/// The returned RecordBatchReader can be closed early to cancel the computation of record
+/// batches. In this case, only errors encountered by the computation may be reported. In
+/// particular, no cancellation error may be reported.
 ARROW_ACERO_EXPORT Result<std::unique_ptr<RecordBatchReader>> DeclarationToReader(
     Declaration declaration, bool use_threads = true,
     MemoryPool* memory_pool = default_memory_pool(),
@@ -746,6 +794,8 @@ ARROW_ACERO_EXPORT Future<> DeclarationToStatusAsync(
 ARROW_ACERO_EXPORT Future<> DeclarationToStatusAsync(Declaration declaration,
                                                      ExecContext exec_context);
 
+/// @}
+
 /// \brief Wrap an ExecBatch generator in a RecordBatchReader.
 ///
 /// The RecordBatchReader does not impose any ordering on emitted batches.
@@ -765,7 +815,5 @@ Result<std::function<Future<std::optional<ExecBatch>>()>> MakeReaderGenerator(
     std::shared_ptr<RecordBatchReader> reader, arrow::internal::Executor* io_executor,
     int max_q = kDefaultBackgroundMaxQ, int q_restart = kDefaultBackgroundQRestart);
 
-/// @}
-
 }  // namespace acero
 }  // namespace arrow