ARROW-10708: [Packaging][deb] Add support for Ubuntu 20.10

dev/release/00-prepare-test.rb

@@ -635,7 +635,7 @@ def test_deb_package_names
           "-Package: libarrow#{@so_version}",
           "+Package: libarrow#{@next_so_version}",
         ],
-        path: "dev/tasks/linux-packages/apache-arrow/debian/control"
+        path: "dev/tasks/linux-packages/apache-arrow/debian/control.in"
       },
       {
         sampled_diff: [

dev/release/00-prepare.sh

@@ -263,9 +263,9 @@ if [ ${PREPARE_DEB_PACKAGE_NAMES} -gt 0 ]; then
 	$(echo $target | sed -e "s/${deb_lib_suffix}/${next_deb_lib_suffix}/")
     done
     deb_lib_suffix_substitute_pattern="s/(lib(arrow|gandiva|parquet|plasma)[-a-z]*)${deb_lib_suffix}/\\1${next_deb_lib_suffix}/g"
-    sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" debian*/control
-    rm -f debian*/control.bak
-    git add debian*/control
+    sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" debian*/control*
+    rm -f debian*/control*.bak
+    git add debian*/control*
     cd -
     cd $SOURCE_DIR/../tasks/
     sed -i.bak -E -e "${deb_lib_suffix_substitute_pattern}" tasks.yml

dev/release/rat_exclude_files.txt

@@ -107,7 +107,7 @@ dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/rules
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/source/format
 dev/tasks/linux-packages/apache-arrow/debian.ubuntu-xenial/watch
 dev/tasks/linux-packages/apache-arrow/debian/compat
-dev/tasks/linux-packages/apache-arrow/debian/control
+dev/tasks/linux-packages/apache-arrow/debian/control.in
 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install
 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install
 dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install
@@ -266,4 +266,4 @@ julia/Arrow/docs/make.jl
 julia/Arrow/docs/mkdocs.yml
 julia/Arrow/docs/src/index.md
 julia/Arrow/docs/src/manual.md
-julia/Arrow/docs/src/reference.md
+julia/Arrow/docs/src/reference.md

dev/tasks/linux-packages/apache-arrow-archive-keyring/apt/ubuntu-groovy/Dockerfile

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM ubuntu:groovy
+
+RUN \
+  echo "debconf debconf/frontend select Noninteractive" | \
+    debconf-set-selections
+
+RUN \
+  echo 'APT::Install-Recommends "false";' > \
+    /etc/apt/apt.conf.d/disable-install-recommends
+
+ARG DEBUG
+
+RUN \
+  quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
+  apt update ${quiet} && \
+  apt install -y -V ${quiet} \
+    build-essential \
+    debhelper \
+    devscripts \
+    fakeroot \
+    gnupg \
+    lsb-release && \
+  apt clean && \
+  rm -rf /var/lib/apt/lists/*

dev/tasks/linux-packages/apache-arrow/Rakefile

@@ -82,6 +82,21 @@ class ApacheArrowPackageTask < PackageTask
          "--output", @full_archive_name)
     end
   end
+
+  def apt_arm64_cuda_available_target?(target)
+    # ubuntu-20.10 has navidia-cuda-toolkit but not libcuda1.
+    # ubuntu-21.04 may support this.
+    false
+  end
+
+  def apt_prepare_debian_control(control_in, target)
+    if apt_arm64_cuda_available_target?(target)
+      cuda_architecture = "any"
+    else
+      cuda_architecture = "i386 amd64"
+    end
+    control_in.gsub(/@CUDA_ARCHITECTURE@/, cuda_architecture)
+  end
 end
 
 task = ApacheArrowPackageTask.new

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy-arm64/from

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+arm64v8/ubuntu:groovy

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/Dockerfile

@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG FROM=ubuntu:groovy
+FROM ${FROM}
+
+COPY qemu-* /usr/bin/
+
+RUN \
+  echo "debconf debconf/frontend select Noninteractive" | \
+    debconf-set-selections
+
+RUN \
+  echo 'APT::Install-Recommends "false";' > \
+    /etc/apt/apt.conf.d/disable-install-recommends
+
+ARG DEBUG
+RUN \
+  quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \
+  apt update ${quiet} && \
+  apt install -y -V ${quiet} \
+    build-essential \
+    clang-11 \
+    cmake \
+    debhelper \
+    devscripts \
+    git \
+    gtk-doc-tools \
+    libboost-filesystem-dev \
+    libboost-regex-dev \
+    libboost-system-dev \
+    libbrotli-dev \
+    libbz2-dev \
+    libcurl4-openssl-dev \
+    libgirepository1.0-dev \
+    libglib2.0-doc \
+    libgmock-dev \
+    libgoogle-glog-dev \
+    libgrpc++-dev \
+    libgtest-dev \
+    liblz4-dev \
+    libre2-dev \
+    libsnappy-dev \
+    libssl-dev \
+    libthrift-dev \
+    libutf8proc-dev \
+    libzstd-dev \
+    llvm-11-dev \
+    lsb-release \
+    ninja-build \
+    pkg-config \
+    protobuf-compiler-grpc \
+    python3-dev \
+    python3-numpy \
+    python3-pip \
+    python3-setuptools \
+    rapidjson-dev \
+    tzdata \
+    zlib1g-dev && \
+  ! apt list | grep -q '^libcuda1' || \
+    apt install -y -V ${quiet} nvidia-cuda-toolkit && \
+  apt clean && \
+  python3 -m pip install --no-use-pep517 meson && \
+  ln -s /usr/local/bin/meson /usr/bin/ && \
+  rm -rf /var/lib/apt/lists/*

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-groovy/qemu-dummy-static

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Do nothing. This exists only for not requiring qemu-aarch64-static copy.
+# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or
+# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian
+# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image.
+#
+# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*",
+# the "COPY" is failed. It means that we always require "qemu*" even if we
+# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file,
+# the "COPY" isn't failed. It means that we can copy "qemu*" only when we
+# need.
+#
+# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml.
+# Azure Pipelines uses old Ubuntu (18.04).
+# So we need to put "qemu-aarch64-static" into this directory.

dev/tasks/linux-packages/apache-arrow/debian/control → dev/tasks/linux-packages/apache-arrow/debian/control.in

@@ -48,7 +48,7 @@ Description: Apache Arrow is a data processing library for analysis
 
 Package: libarrow-cuda300
 Section: libs
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
@@ -128,7 +128,7 @@ Description: Apache Arrow is a data processing library for analysis
 
 Package: libarrow-cuda-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${misc:Depends},
@@ -217,7 +217,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libplasma300
 Section: libs
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
@@ -230,7 +230,7 @@ Description: Plasma is an in-memory object store and cache for big data.
 
 Package: plasma-store-server
 Section: utils
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
   ${misc:Depends},
@@ -242,7 +242,7 @@ Description: Plasma is an in-memory object store and cache for big data.
 
 Package: libplasma-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${misc:Depends},
@@ -328,7 +328,7 @@ Description: Apache Arrow is a data processing library for analysis
 
 Package: libarrow-cuda-glib300
 Section: libs
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
@@ -342,7 +342,7 @@ Description: Apache Arrow is a data processing library for analysis
 
 Package: gir1.2-arrow-cuda-1.0
 Section: introspection
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${gir:Depends},
@@ -353,7 +353,7 @@ Description: Apache Arrow is a data processing library for analysis
 
 Package: libarrow-cuda-glib-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${misc:Depends},
@@ -471,7 +471,7 @@ Description: Gandiva is a toolset for compiling and evaluating expressions
 
 Package: libplasma-glib300
 Section: libs
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Pre-Depends: ${misc:Pre-Depends}
 Depends:
@@ -485,7 +485,7 @@ Description: Plasma is an in-memory object store and cache for big data.
 
 Package: gir1.2-plasma-1.0
 Section: introspection
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${gir:Depends},
@@ -496,7 +496,7 @@ Description: Plasma is an in-memory object store and cache for big data.
 
 Package: libplasma-glib-dev
 Section: libdevel
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: same
 Depends:
   ${misc:Depends},
@@ -510,7 +510,7 @@ Description: Plasma is an in-memory object store and cache for big data.
 
 Package: libplasma-glib-doc
 Section: doc
-Architecture: i386 amd64
+Architecture: @CUDA_ARCHITECTURE@
 Multi-Arch: foreign
 Depends:
   ${misc:Depends}