ARROW-14061: [Go][C++] Add Cgo Arrow Memory Pool Allocator

.github/workflows/go.yml

@@ -75,6 +75,40 @@ jobs:
         continue-on-error: true
         run: archery docker push debian-go
 
+  docker_cgo:
+    name: AMD64 Debian 10 GO ${{ matrix.go }} - CGO
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [1.15]
+    env:
+      GO: ${{ matrix.go }}
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        run: ci/scripts/util_checkout.sh
+      - name: Free Up Disk Space
+        run: ci/scripts/util_cleanup.sh
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Setup Archery
+        run: pip install -e dev/archery[docker]
+      - name: Execute Docker Build
+        run: archery docker run debian-go-cgo
+      - name: Docker Push
+        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
+        continue-on-error: true
+        run: archery docker push debian-go-cgo
+
+
   docker_cgo_python:
     name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python
     runs-on: ubuntu-latest
@@ -163,3 +197,91 @@ jobs:
       - name: Test
         shell: bash
         run: ci/scripts/go_test.sh .
+
+  macos-cgo:
+    name: AMD64 MacOS 10.15 Go ${{ matrix.go }} - CGO
+    runs-on: macos-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [1.15]
+    env:
+      ARROW_GO_TESTCGO: "1"
+    steps:
+      - name: Install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        shell: bash
+        run: ci/scripts/util_checkout.sh
+      - name: Brew Install Arrow
+        shell: bash
+        run: brew install apache-arrow
+      - name: Build
+        shell: bash
+        run: ci/scripts/go_build.sh .
+      - name: Test
+        shell: bash
+        run: ci/scripts/go_test.sh .
+
+  windows-mingw:
+    name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO
+    runs-on: windows-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        go: [1.15]
+        mingw-n-bits:
+          #- 32 runtime handling for CGO needs 64-bit currently
+          - 64
+    env:
+      ARROW_GO_TESTCGO: "1"            
+    steps:
+      - name: Disable Crash Dialogs
+        run: |
+          reg add `
+            "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" `
+            /v DontShowUI `
+            /t REG_DWORD `
+            /d 1 `
+            /f
+      - name: Install go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ matrix.go }}
+      - name: Checkout Arrow
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Fetch Submodules and Tags
+        shell: bash
+        run: ci/scripts/util_checkout.sh
+      - uses: msys2/setup-msys2@v2
+        with:
+          msystem: MINGW${{ matrix.mingw-n-bits }}
+          update: true
+      - name: Setup MSYS2
+        shell: msys2 {0}
+        run: |
+          ci/scripts/msys2_setup.sh cgo
+      - name: Update CGO Env vars
+        shell: msys2 {0}
+        run: |
+          echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV
+          echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV
+          echo "$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_PATH
+      - name: Build
+        shell: bash
+        run: ci/scripts/go_build.sh .
+      - name: Test
+        shell: bash
+        run: ci/scripts/go_test.sh .

ci/docker/debian-10-go-cgo.dockerfile

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+ENV DEBIAN_FRONTEND noninteractive
+
+# install libarrow-dev and libarrow-python-dev so we can use pyarrow in some tests
+# in order to test the C Data Interface
+RUN apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends ca-certificates lsb-release wget && \
+    wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+    apt-get install -y -q --no-install-recommends ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
+    apt-get update -y -q && \
+    apt-get install -y -q --no-install-recommends \
+        cmake \
+        libarrow-dev \
+        libarrow-python-dev && \
+    apt-get clean

ci/scripts/go_build.sh

@@ -23,8 +23,12 @@ source_dir=${1}/go
 
 pushd ${source_dir}/arrow
 
+if [[ -n "${ARROW_GO_TESTCGO}" ]]; then
+    TAGS="-tags ccalloc"
+fi
+
 go get -d -t -v ./...
-go install -v ./...
+go install $TAGS -v ./...
 
 popd
 

ci/scripts/go_test.sh

@@ -31,12 +31,18 @@ esac
 
 pushd ${source_dir}/arrow
 
+TAGS="test"
+if [[ -n "${ARROW_GO_TESTCGO}" ]]; then
+    TAGS="${TAGS},ccalloc"
+fi
+
+
 # the cgo implementation of the c data interface requires the "test"
 # tag in order to run its tests so that the testing functions implemented
 # in .c files don't get included in non-test builds.
 
 for d in $(go list ./... | grep -v vendor); do
-    go test $testargs -tags "test" $d
+    go test $testargs -tags $TAGS $d
 done
 
 popd

ci/scripts/msys2_setup.sh

@@ -60,6 +60,13 @@ case "${target}" in
     ;;
 esac
 
+case "${target}" in 
+  cgo)
+    packages+=(${MINGW_PACKAGE_PREFIX}-arrow)
+    packages+=(${MINGW_PACKAGE_PREFIX}-gcc)
+    ;;
+esac
+
 pacman \
   --needed \
   --noconfirm \

docker-compose.yml

@@ -113,6 +113,7 @@ x-hierarchy:
       - debian-ruby
     - debian-python
   - debian-go:
+    - debian-go-cgo
     - debian-go-cgo-python
   - debian-java:
     - debian-java-jni
@@ -1199,6 +1200,24 @@ services:
         /arrow/ci/scripts/go_build.sh /arrow &&
         /arrow/ci/scripts/go_test.sh /arrow"
 
+  debian-go-cgo:
+    # Usage:
+    #   docker-compose build debian-go-cgo
+    #   docker-compose run debian-go-cgo
+    image: ${REPO}:${ARCH}-debian-10-go-${GO}-cgo
+    build:
+      context: .
+      dockerfile: ci/docker/debian-10-go-cgo.dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-debian-10-go-${GO}-cgo
+      args:
+        base: ${REPO}:${ARCH}-debian-10-go-${GO}
+    shm_size: *shm-size
+    volumes: *debian-volumes
+    environment:
+      ARROW_GO_TESTCGO: "1"
+    command: *go-command
+
   debian-go-cgo-python:
     # Usage:
     #   docker-compose build debian-go-cgo-python

go/arrow/cdata/cdata.go

@@ -22,9 +22,6 @@ package cdata
 
 // #include "arrow/c/abi.h"
 // #include "arrow/c/helpers.h"
-// typedef struct ArrowSchema ArrowSchema;
-// typedef struct ArrowArray ArrowArray;
-// typedef struct ArrowArrayStream ArrowArrayStream;
 //
 // int stream_get_schema(struct ArrowArrayStream* st, struct ArrowSchema* out) { return st->get_schema(st, out); }
 // int stream_get_next(struct ArrowArrayStream* st, struct ArrowArray* out) { return st->get_next(st, out); }
@@ -52,12 +49,12 @@ import (
 
 type (
 	// CArrowSchema is the C Data Interface for ArrowSchemas defined in abi.h
-	CArrowSchema = C.ArrowSchema
+	CArrowSchema = C.struct_ArrowSchema
 	// CArrowArray is the C Data Interface object for Arrow Arrays as defined in abi.h
-	CArrowArray = C.ArrowArray
+	CArrowArray = C.struct_ArrowArray
 	// CArrowArrayStream is the Experimental API for handling streams of record batches
 	// through the C Data interface.
-	CArrowArrayStream = C.ArrowArrayStream
+	CArrowArrayStream = C.struct_ArrowArrayStream
 )
 
 // Map from the defined strings to their corresponding arrow.DataType interface
@@ -146,7 +143,7 @@ func importSchema(schema *CArrowSchema) (ret arrow.Field, err error) {
 	var childFields []arrow.Field
 	if schema.n_children > 0 {
 		// call ourselves recursively if there are children.
-		var schemaChildren []*C.ArrowSchema
+		var schemaChildren []*CArrowSchema
 		// set up a slice to reference safely
 		s := (*reflect.SliceHeader)(unsafe.Pointer(&schemaChildren))
 		s.Data = uintptr(unsafe.Pointer(schema.children))
@@ -255,21 +252,21 @@ func importSchema(schema *CArrowSchema) (ret arrow.Field, err error) {
 // importer to keep track when importing C ArrowArray objects.
 type cimporter struct {
 	dt       arrow.DataType
-	arr      *C.ArrowArray
+	arr      *CArrowArray
 	data     *array.Data
 	parent   *cimporter
 	children []cimporter
 	cbuffers []*C.void
 }
 
-func (imp *cimporter) importChild(parent *cimporter, src *C.ArrowArray) error {
+func (imp *cimporter) importChild(parent *cimporter, src *CArrowArray) error {
 	imp.parent = parent
 	return imp.doImport(src)
 }
 
 // import any child arrays for lists, structs, and so on.
 func (imp *cimporter) doImportChildren() error {
-	var children []*C.ArrowArray
+	var children []*CArrowArray
 	// create a proper slice for our children
 	s := (*reflect.SliceHeader)(unsafe.Pointer(&children))
 	s.Data = uintptr(unsafe.Pointer(imp.arr.children))
@@ -315,13 +312,13 @@ func (imp *cimporter) initarr() {
 
 // import is called recursively as needed for importing an array and its children
 // in order to generate array.Data objects
-func (imp *cimporter) doImport(src *C.ArrowArray) error {
+func (imp *cimporter) doImport(src *CArrowArray) error {
 	imp.initarr()
 	// move the array from the src object passed in to the one referenced by
 	// this importer. That way we can set up a finalizer on the created
 	// *array.Data object so we clean up our Array's memory when garbage collected.
 	C.ArrowArrayMove(src, imp.arr)
-	defer func(arr *C.ArrowArray) {
+	defer func(arr *CArrowArray) {
 		if imp.data != nil {
 			runtime.SetFinalizer(imp.data, func(*array.Data) {
 				C.ArrowArrayRelease(arr)
@@ -339,7 +336,9 @@ func (imp *cimporter) doImport(src *C.ArrowArray) error {
 
 	// get a view of the buffers, zero-copy. we're just looking at the pointers
 	const maxlen = 0x7fffffff
-	imp.cbuffers = (*[maxlen]*C.void)(unsafe.Pointer(imp.arr.buffers))[:imp.arr.n_buffers:imp.arr.n_buffers]
+	if imp.arr.n_buffers > 0 {
+		imp.cbuffers = (*[maxlen]*C.void)(unsafe.Pointer(imp.arr.buffers))[:imp.arr.n_buffers:imp.arr.n_buffers]
+	}
 
 	// handle each of our type cases
 	switch dt := imp.dt.(type) {
@@ -521,13 +520,13 @@ func (imp *cimporter) importVariableValuesBuffer(bufferID int, byteWidth int, of
 	return imp.importBuffer(bufferID, int64(bufsize))
 }
 
-func importCArrayAsType(arr *C.ArrowArray, dt arrow.DataType) (imp *cimporter, err error) {
+func importCArrayAsType(arr *CArrowArray, dt arrow.DataType) (imp *cimporter, err error) {
 	imp = &cimporter{dt: dt}
 	err = imp.doImport(arr)
 	return
 }
 
-func initReader(rdr *nativeCRecordBatchReader, stream *C.ArrowArrayStream) {
+func initReader(rdr *nativeCRecordBatchReader, stream *CArrowArrayStream) {
 	st := C.get_stream()
 	rdr.stream = &st
 	C.ArrowArrayStreamMove(stream, rdr.stream)
@@ -536,7 +535,7 @@ func initReader(rdr *nativeCRecordBatchReader, stream *C.ArrowArrayStream) {
 
 // Record Batch reader that conforms to arrio.Reader for the ArrowArrayStream interface
 type nativeCRecordBatchReader struct {
-	stream *C.ArrowArrayStream
+	stream *CArrowArrayStream
 	schema *arrow.Schema
 }
 
@@ -546,7 +545,7 @@ func (n *nativeCRecordBatchReader) getError(errno int) error {
 
 func (n *nativeCRecordBatchReader) Read() (array.Record, error) {
 	if n.schema == nil {
-		var sc C.ArrowSchema
+		var sc CArrowSchema
 		errno := C.stream_get_schema(n.stream, &sc)
 		if errno != 0 {
 			return nil, n.getError(int(errno))