ARROW-13816: [Go][C] Implement Consumer APIs for C Data Interface in Go

.github/workflows/go.yml

@@ -75,6 +75,39 @@ jobs:
         continue-on-error: true
         run: archery docker push debian-go
 
+  docker_cgo_python:
+    name: AMD64 Debian 10 GO ${{ matrix.go }} - CGO Python
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [1.15]
+    env:
+      GO: ${{ matrix.go }}
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        run: ci/scripts/util_checkout.sh
+      - name: Free Up Disk Space
+        run: ci/scripts/util_cleanup.sh
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Setup Archery
+        run: pip install -e dev/archery[docker]
+      - name: Execute Docker Build
+        run: archery docker run debian-go-cgo-python
+      - name: Docker Push
+        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        continue-on-error: true
+        run: archery docker push debian-go-cgo-python
+
   windows:
     name: AMD64 Windows 2019 Go ${{ matrix.go }}
     runs-on: windows-latest

ci/docker/debian-10-go-cgo-python.dockerfile

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install python3 and pip so we can install pyarrow to test the C data interface.
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        python3 \
+        python3-pip && \
+    apt-get clean
+
+RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
+    ln -s /usr/bin/pip3 /usr/local/bin/pip
+
+# Need a newer pip than Debian's to install manylinux201x wheels
+RUN pip install -U pip
+
+RUN pip install pyarrow cffi --only-binary pyarrow

ci/docker/debian-10-go.dockerfile

@@ -17,7 +17,8 @@
 
 ARG arch=amd64
 ARG go=1.15
-FROM ${arch}/golang:${go}
+FROM ${arch}/golang:${go}-buster
+
 
 # TODO(kszucs):
 # 1. add the files required to install the dependencies to .dockerignore

ci/scripts/go_cgo_python_test.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+source_dir=${1}/go
+
+pushd ${source_dir}/arrow/cdata/test
+
+case "$(uname)" in
+    Linux)
+        testlib="cgotest.so"
+        ;;
+    Darwin)
+        testlib="cgotest.so"
+        ;;
+    MINGW*)
+        testlib="cgotest.dll"
+        ;;
+esac
+
+go build -tags cdata_test -buildmode=c-shared -o $testlib .
+
+python test_export_to_cgo.py
+
+rm $testlib
+rm "${testlib%.*}.h"
+
+popd

ci/scripts/go_test.sh

@@ -31,6 +31,10 @@ esac
 
 pushd ${source_dir}/arrow
 
+# the cgo implementation of the c data interface requires the "test"
+# tag in order to run its tests so that the testing functions implemented
+# in .c files don't get included in non-test builds.
+
 for d in $(go list ./... | grep -v vendor); do
     go test $testargs -tags "test" $d
 done

dev/release/rat_exclude_files.txt

@@ -129,6 +129,7 @@ go/arrow/flight/Flight.pb.go
 go/arrow/flight/Flight_grpc.pb.go
 go/arrow/internal/cpu/*
 go/arrow/type_string.go
+go/arrow/cdata/test/go.sum
 go/*.tmpldata
 go/*.s
 go/parquet/go.sum

docker-compose.yml

@@ -112,7 +112,8 @@ x-hierarchy:
     - debian-c-glib:
       - debian-ruby
     - debian-python
-  - debian-go
+  - debian-go:
+    - debian-go-cgo-python
   - debian-java:
     - debian-java-jni
   - debian-js
@@ -1198,6 +1199,24 @@ services:
         /arrow/ci/scripts/go_build.sh /arrow &&
         /arrow/ci/scripts/go_test.sh /arrow"
 
+  debian-go-cgo-python:
+    # Usage:
+    #   docker-compose build debian-go-cgo-python
+    #   docker-compose run debian-go-cgo-python
+    image: ${REPO}:${ARCH}-debian-10-go-${GO}-cgo-python
+    build:
+      context: .
+      dockerfile: ci/docker/debian-10-go-cgo-python.dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-debian-10-go-${GO}-cgo-python
+      args:
+        base: ${REPO}:${ARCH}-debian-10-go-${GO}
+    shm_size: *shm-size
+    volumes: *debian-volumes
+    command: &go-cgo-python-command >
+        /bin/bash -c "
+          /arrow/ci/scripts/go_cgo_python_test.sh /arrow"
+
   ############################# JavaScript ####################################
 
   debian-js:

docs/source/status.rst

@@ -177,18 +177,18 @@ Notes:
 C Data Interface
 ================
 
-+-----------------------------+-------+--------+-------+-------+
-| Feature                     | C++   | Python | R     | Rust  |
-|                             |       |        |       |       |
-+=============================+=======+========+=======+=======+
-| Schema export               | ✓     | ✓      | ✓     | ✓     |
-+-----------------------------+-------+--------+-------+-------+
-| Array export                | ✓     | ✓      | ✓     | ✓     |
-+-----------------------------+-------+--------+-------+-------+
-| Schema import               | ✓     | ✓      | ✓     | ✓     |
-+-----------------------------+-------+--------+-------+-------+
-| Array import                | ✓     | ✓      | ✓     | ✓     |
-+-----------------------------+-------+--------+-------+-------+
++-----------------------------+-------+--------+-------+-------+----+
+| Feature                     | C++   | Python | R     | Rust  | Go |
+|                             |       |        |       |       |    |
++=============================+=======+========+=======+=======+====+
+| Schema export               | ✓     | ✓      | ✓    | ✓     |    |
++-----------------------------+-------+--------+-------+-------+----+
+| Array export                | ✓     | ✓      | ✓    | ✓     |    |
++-----------------------------+-------+--------+-------+-------+----+
+| Schema import               | ✓     | ✓      | ✓    | ✓     | ✓  |
++-----------------------------+-------+--------+-------+-------+-----+
+| Array import                | ✓     | ✓      | ✓    | ✓     | ✓  |
++-----------------------------+-------+--------+-------+-------+-----+
 
 .. seealso::
    The :ref:`C Data Interface <c-data-interface>` specification.
@@ -197,14 +197,14 @@ C Data Interface
 C Stream Interface (experimental)
 =================================
 
-+-----------------------------+-------+--------+
-| Feature                     | C++   | Python |
-|                             |       |        |
-+=============================+=======+========+
-| Stream export               | ✓     | ✓      |
-+-----------------------------+-------+--------+
-| Stream import               | ✓     | ✓      |
-+-----------------------------+-------+--------+
++-----------------------------+-------+--------+----+
+| Feature                     | C++   | Python | Go |
+|                             |       |        |    |
++=============================+=======+========+====+
+| Stream export               | ✓     | ✓      |   |
++-----------------------------+-------+--------+----+
+| Stream import               | ✓     | ✓      | ✓ |
++-----------------------------+-------+--------+----+
 
 .. seealso::
    The :ref:`C Stream Interface <c-stream-interface>` specification.

go/arrow/cdata/arrow/c/abi.h

@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define ARROW_FLAG_DICTIONARY_ORDERED 1
+#define ARROW_FLAG_NULLABLE 2
+#define ARROW_FLAG_MAP_KEYS_SORTED 4
+
+struct ArrowSchema {
+  // Array type description
+  const char* format;
+  const char* name;
+  const char* metadata;
+  int64_t flags;
+  int64_t n_children;
+  struct ArrowSchema** children;
+  struct ArrowSchema* dictionary;
+
+  // Release callback
+  void (*release)(struct ArrowSchema*);
+  // Opaque producer-specific data
+  void* private_data;
+};
+
+struct ArrowArray {
+  // Array data description
+  int64_t length;
+  int64_t null_count;
+  int64_t offset;
+  int64_t n_buffers;
+  int64_t n_children;
+  const void** buffers;
+  struct ArrowArray** children;
+  struct ArrowArray* dictionary;
+
+  // Release callback
+  void (*release)(struct ArrowArray*);
+  // Opaque producer-specific data
+  void* private_data;
+};
+
+// EXPERIMENTAL: C stream interface
+
+struct ArrowArrayStream {
+  // Callback to get the stream type
+  // (will be the same for all arrays in the stream).
+  //
+  // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+  //
+  // If successful, the ArrowSchema must be released independently from the stream.
+  int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out);
+
+  // Callback to get the next array
+  // (if no error and the array is released, the stream has ended)
+  //
+  // Return value: 0 if successful, an `errno`-compatible error code otherwise.
+  //
+  // If successful, the ArrowArray must be released independently from the stream.
+  int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out);
+
+  // Callback to get optional detailed error information.
+  // This must only be called if the last stream operation failed
+  // with a non-0 return code.
+  //
+  // Return value: pointer to a null-terminated character array describing
+  // the last error, or NULL if no description is available.
+  //
+  // The returned pointer is only valid until the next operation on this stream
+  // (including release).
+  const char* (*get_last_error)(struct ArrowArrayStream*);
+
+  // Release callback: release the stream's own resources.
+  // Note that arrays returned by `get_next` must be individually released.
+  void (*release)(struct ArrowArrayStream*);
+
+  // Opaque producer-specific data
+  void* private_data;
+};
+
+#ifdef __cplusplus
+}
+#endif