diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 2c5f2965107b8..0000000000000 --- a/.travis.yml +++ /dev/null @@ -1,166 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -dist: focal - -language: minimal - -cache: - directories: - - $TRAVIS_BUILD_DIR/.docker - -addons: - apt: - packages: - - python3-pip - -services: - - docker - -# Note that the global "env" setting isn't inherited automatically by -# matrix entries with their own "env", so we have to insert it explicitly. -env: &global_env - ARROW_ENABLE_TIMING_TESTS: "OFF" - COMPOSE_DOCKER_CLI_BUILD: 1 - DOCKER_BUILDKIT: 0 - DOCKER_VOLUME_PREFIX: $TRAVIS_BUILD_DIR/.docker/ - -jobs: - include: - - name: "C++ on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "CPP" - DOCKER_IMAGE_ID: ubuntu-cpp - # Can't enable ARROW_MIMALLOC because of failures in memory pool tests. - # Can't enable ARROW_S3 because compiler is killed while compiling - # aws-sdk-cpp. - DOCKER_RUN_ARGS: >- - " - -e ARROW_FLIGHT=ON - -e ARROW_GCS=OFF - -e ARROW_MIMALLOC=OFF - -e ARROW_ORC=OFF - -e ARROW_PARQUET=OFF - -e ARROW_S3=OFF - -e ARROW_SUBSTRAIT=OFF - -e CMAKE_BUILD_PARALLEL_LEVEL=2 - -e CMAKE_UNITY_BUILD=ON - -e PARQUET_BUILD_EXAMPLES=OFF - -e PARQUET_BUILD_EXECUTABLES=OFF - -e Protobuf_SOURCE=BUNDLED - -e gRPC_SOURCE=BUNDLED - " - # The LLVM's APT repository causes download error for s390x binary - # We should use the LLVM provided by the default APT repository - CLANG_TOOLS: "10" - LLVM: "10" - UBUNTU: "20.04" - - - name: "Go on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "GO" - DOCKER_IMAGE_ID: debian-go - - - name: "Java on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "JAVA" - DOCKER_IMAGE_ID: debian-java - JDK: 11 - - - name: "Python on s390x" - os: linux - arch: s390x - env: - <<: *global_env - ARCH: s390x - ARROW_CI_MODULES: "PYTHON" - DOCKER_IMAGE_ID: ubuntu-python - # Can't enable ARROW_MIMALLOC because of failures in memory pool tests. - # Can't enable ARROW_S3 because compiler is killed while compiling - # aws-sdk-cpp. - DOCKER_RUN_ARGS: >- - " - -e ARROW_FLIGHT=ON - -e ARROW_GCS=OFF - -e ARROW_MIMALLOC=OFF - -e ARROW_ORC=OFF - -e ARROW_PARQUET=OFF - -e ARROW_PYTHON=ON - -e ARROW_S3=OFF - -e CMAKE_BUILD_PARALLEL_LEVEL=2 - -e CMAKE_UNITY_BUILD=ON - -e PARQUET_BUILD_EXAMPLES=OFF - -e PARQUET_BUILD_EXECUTABLES=OFF - -e Protobuf_SOURCE=BUNDLED - -e gRPC_SOURCE=BUNDLED - " - # The LLVM's APT repository causes download error for s390x binary - # We should use the LLVM provided by the default APT repository - CLANG_TOOLS: "10" - LLVM: "10" - UBUNTU: "20.04" - - allow_failures: - - name: "Java on s390x" - - name: "C++ on s390x" - - name: "Python on s390x" - -before_install: - - eval "$(python ci/detect-changes.py)" - - | - arrow_ci_affected=no - for arrow_ci_module in ${ARROW_CI_MODULES}; do - arrow_ci_affected_variable=ARROW_CI_${arrow_ci_module}_AFFECTED - if [ "$(eval "echo \$${arrow_ci_affected_variable}")" = "1" ]; then - arrow_ci_affected=yes - fi - done - if [ "${arrow_ci_affected}" = "no" ]; then - travis_terminate 0 - fi - -install: - - sudo -H pip3 install --upgrade pip - - sudo -H pip3 install 'docker-compose>=1.27.0' - - sudo -H pip3 install -e dev/archery[docker] - -script: - - export ARCHERY_DEFAULT_BRANCH=$(git rev-parse --abbrev-ref origin/HEAD | sed s@origin/@@) - - | - archery docker run \ - ${DOCKER_RUN_ARGS} \ - --volume ${PWD}/build:/build \ - ${DOCKER_IMAGE_ID} - -after_success: - - | - if [ "${TRAVIS_EVENT_TYPE}" = "push" -a \ - "${TRAVIS_REPO_SLUG}" = "apache/arrow" ]; then - archery docker push ${DOCKER_IMAGE_ID} || : - fi diff --git a/ci/detect-changes.py b/ci/detect-changes.py deleted file mode 100644 index 7669639ecd3a9..0000000000000 --- a/ci/detect-changes.py +++ /dev/null @@ -1,362 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import print_function - -import functools -import os -import pprint -import re -import sys -import subprocess - - -perr = functools.partial(print, file=sys.stderr) - - -def dump_env_vars(prefix, pattern=None): - if pattern is not None: - match = lambda s: re.search(pattern, s) - else: - match = lambda s: True - for name in sorted(os.environ): - if name.startswith(prefix) and match(name): - perr("- {0}: {1!r}".format(name, os.environ[name])) - - -def run_cmd(cmdline): - proc = subprocess.Popen(cmdline, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = proc.communicate() - if proc.returncode != 0: - raise RuntimeError("Command {cmdline} failed with code {returncode}, " - "stderr was:\n{stderr}\n" - .format(cmdline=cmdline, returncode=proc.returncode, - stderr=err.decode())) - return out - - -def get_commit_description(commit): - """ - Return the textual description (title + body) of the given git commit. - """ - out = run_cmd(["git", "show", "--no-patch", "--pretty=format:%B", - commit]) - return out.decode('utf-8', 'ignore') - - -def list_affected_files(commit_range): - """ - Return a list of files changed by the given git commit range. - """ - perr("Getting affected files from", repr(commit_range)) - out = run_cmd(["git", "diff", "--name-only", commit_range]) - return list(filter(None, (s.strip() for s in out.decode().splitlines()))) - - -def get_travis_head_commit(): - return os.environ['TRAVIS_COMMIT'] - - -def get_travis_commit_range(): - if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request': - # TRAVIS_COMMIT_RANGE is too pessimistic for PRs, as it may contain - # unrelated changes. Instead, use the same strategy as on AppVeyor - # below. - run_cmd(["git", "fetch", "-q", "origin", - "+refs/heads/{0}".format(os.environ['TRAVIS_BRANCH'])]) - merge_base = run_cmd(["git", "merge-base", - "HEAD", "FETCH_HEAD"]).decode().strip() - return "{0}..HEAD".format(merge_base) - else: - cr = os.environ['TRAVIS_COMMIT_RANGE'] - # See - # https://github.com/travis-ci/travis-ci/issues/4596#issuecomment-139811122 - return cr.replace('...', '..') - - -def get_travis_commit_description(): - # Prefer this to get_commit_description(get_travis_head_commit()), - # as rebasing or other repository events may make TRAVIS_COMMIT invalid - # at the time we inspect it - return os.environ['TRAVIS_COMMIT_MESSAGE'] - - -def list_travis_affected_files(): - """ - Return a list of files affected in the current Travis build. - """ - commit_range = get_travis_commit_range() - try: - return list_affected_files(commit_range) - except RuntimeError: - # TRAVIS_COMMIT_RANGE can contain invalid revisions when - # building a branch (not a PR) after rebasing: - # https://github.com/travis-ci/travis-ci/issues/2668 - if os.environ['TRAVIS_EVENT_TYPE'] == 'pull_request': - raise - # If it's a rebase, it's probably enough to use the last commit only - commit_range = '{0}^..'.format(get_travis_head_commit()) - return list_affected_files(commit_range) - - -def list_appveyor_affected_files(): - """ - Return a list of files affected in the current AppVeyor build. - This only works for PR builds. - """ - # Re-fetch PR base branch (e.g. origin/master), pointing FETCH_HEAD to it - run_cmd(["git", "fetch", "-q", "origin", - "+refs/heads/{0}".format(os.environ['APPVEYOR_REPO_BRANCH'])]) - # Compute base changeset between FETCH_HEAD (PR base) and HEAD (PR head) - merge_base = run_cmd(["git", "merge-base", - "HEAD", "FETCH_HEAD"]).decode().strip() - # Compute changes files between base changeset and HEAD - return list_affected_files("{0}..HEAD".format(merge_base)) - - -def list_github_actions_affected_files(): - """ - Return a list of files affected in the current GitHub Actions build. - """ - # GitHub Actions checkout `refs/remotes/pull/$PR/merge` where `HEAD` points - # to the merge commit while `HEAD^` points to the commit before. Hence, - # `..HEAD^` points to all commit between the default branch and the PR. - return list_affected_files("HEAD^..") - - -LANGUAGE_TOPICS = ['c_glib', 'cpp', 'docs', 'go', 'java', 'js', 'python', - 'r', 'ruby', 'csharp'] - -ALL_TOPICS = LANGUAGE_TOPICS + ['integration', 'dev'] - - -AFFECTED_DEPENDENCIES = { - 'java': ['integration', 'python'], - 'js': ['integration'], - 'ci': ALL_TOPICS, - 'cpp': ['python', 'c_glib', 'r', 'ruby', 'integration'], - 'format': LANGUAGE_TOPICS, - 'go': ['integration'], - '.travis.yml': ALL_TOPICS, - 'appveyor.yml': ALL_TOPICS, - # In theory, it should ignore CONTRIBUTING.md and ISSUE_TEMPLATE.md, but in - # practice it's going to be CI - '.github': ALL_TOPICS, - 'c_glib': ['ruby'] -} - -COMPONENTS = {'cpp', 'java', 'c_glib', 'r', 'ruby', 'integration', 'js', - 'csharp', 'go', 'docs', 'python', 'dev'} - - -def get_affected_topics(affected_files): - """ - Return a dict of topics affected by the given files. - Each dict value is True if affected, False otherwise. - """ - affected = dict.fromkeys(ALL_TOPICS, False) - - for path in affected_files: - parts = [] - head = path - while head: - head, tail = os.path.split(head) - parts.append(tail) - parts.reverse() - assert parts - p = parts[0] - fn = parts[-1] - if fn.startswith('README'): - continue - - if p in COMPONENTS: - affected[p] = True - - _path_already_affected = {} - - def _affect_dependencies(component): - if component in _path_already_affected: - # For circular dependencies, terminate - return - for topic in AFFECTED_DEPENDENCIES.get(component, ()): - affected[topic] = True - _affect_dependencies(topic) - _path_already_affected[topic] = True - - _affect_dependencies(p) - - return affected - - -def make_env_for_topics(affected): - return {'ARROW_CI_{0}_AFFECTED'.format(k.upper()): '1' if v else '0' - for k, v in affected.items()} - - -def get_unix_shell_eval(env): - """ - Return a shell-evalable string to setup some environment variables. - """ - return "; ".join(("export {0}='{1}'".format(k, v) - for k, v in env.items())) - - -def get_windows_shell_eval(env): - """ - Return a shell-evalable string to setup some environment variables. - """ - return "\n".join(('set "{0}={1}"'.format(k, v) - for k, v in env.items())) - - -def run_from_travis(): - perr("Environment variables (excerpt):") - dump_env_vars('TRAVIS_', '(BRANCH|COMMIT|PULL)') - if (os.environ['TRAVIS_REPO_SLUG'] == 'apache/arrow' and - os.environ['TRAVIS_BRANCH'] in ['master', 'main'] and - os.environ['TRAVIS_EVENT_TYPE'] != 'pull_request'): - # Never skip anything on default-branch builds in the official repo - affected = dict.fromkeys(ALL_TOPICS, True) - else: - desc = get_travis_commit_description() - if '[skip travis]' in desc: - # Skip everything - affected = dict.fromkeys(ALL_TOPICS, False) - elif '[force ci]' in desc or '[force travis]' in desc: - # Test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - # Test affected topics - affected_files = list_travis_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_unix_shell_eval(make_env_for_topics(affected)) - - -def run_from_appveyor(): - perr("Environment variables (excerpt):") - dump_env_vars('APPVEYOR_', '(PULL|REPO)') - if not os.environ.get('APPVEYOR_PULL_REQUEST_HEAD_COMMIT'): - # Not a PR build, test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - affected_files = list_appveyor_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_windows_shell_eval(make_env_for_topics(affected)) - - -def run_from_github(): - perr("Environment variables (excerpt):") - dump_env_vars('GITHUB_', '(REPOSITORY|ACTOR|SHA|REF|HEAD_REF|BASE_REF|EVENT_NAME)') - if os.environ['GITHUB_EVENT_NAME'] != 'pull_request': - # Not a PR build, test everything - affected = dict.fromkeys(ALL_TOPICS, True) - else: - affected_files = list_github_actions_affected_files() - perr("Affected files:", affected_files) - affected = get_affected_topics(affected_files) - assert set(affected) <= set(ALL_TOPICS), affected - - perr("Affected topics:") - perr(pprint.pformat(affected)) - return get_unix_shell_eval(make_env_for_topics(affected)) - - -def test_get_affected_topics(): - affected_topics = get_affected_topics(['cpp/CMakeLists.txt']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': False, - 'go': False, - 'java': False, - 'js': False, - 'python': True, - 'r': True, - 'ruby': True, - 'csharp': False, - 'integration': True, - 'dev': False - } - - affected_topics = get_affected_topics(['format/Schema.fbs']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': True, - 'go': True, - 'java': True, - 'js': True, - 'python': True, - 'r': True, - 'ruby': True, - 'csharp': True, - 'integration': True, - 'dev': False - } - - affected_topics = get_affected_topics(['.github/workflows']) - assert affected_topics == { - 'c_glib': True, - 'cpp': True, - 'docs': True, - 'go': True, - 'java': True, - 'js': True, - 'python': True, - 'r': True, - 'ruby': True, - 'csharp': True, - 'integration': True, - 'dev': True, - } - - -if __name__ == "__main__": - # This script should have its output evaluated by a shell, - # e.g. "eval `python ci/detect-changes.py`" - if os.environ.get('TRAVIS'): - try: - print(run_from_travis()) - except Exception: - # Make sure the enclosing eval will return an error - print("exit 1") - raise - elif os.environ.get('APPVEYOR'): - try: - print(run_from_appveyor()) - except Exception: - print("exit 1") - raise - elif os.environ.get('GITHUB_WORKFLOW'): - try: - print(run_from_github()) - except Exception: - print("exit 1") - raise - else: - sys.exit("Script must be run under Travis-CI, AppVeyor or GitHub Actions") diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile index 19f30717ca2e2..c59766c4a665c 100644 --- a/ci/docker/linux-apt-r.dockerfile +++ b/ci/docker/linux-apt-r.dockerfile @@ -47,10 +47,6 @@ RUN apt-get update -y && \ libxml2-dev \ libgit2-dev \ libssl-dev \ - # install clang to mirror what was done on Travis - clang \ - clang-format \ - clang-tidy \ # R CMD CHECK --as-cran needs pdflatex to build the package manual texlive-latex-base \ # Need locales so we can set UTF-8 diff --git a/docs/source/developers/continuous_integration/crossbow.rst b/docs/source/developers/continuous_integration/crossbow.rst index 663fc17c0a028..6308f077ac9a6 100644 --- a/docs/source/developers/continuous_integration/crossbow.rst +++ b/docs/source/developers/continuous_integration/crossbow.rst @@ -47,7 +47,7 @@ Executors Individual jobs are executed on public CI services, currently: - Linux: GitHub Actions, Travis CI, Azure Pipelines -- macOS: GitHub Actions, Travis CI, Azure Pipelines +- macOS: GitHub Actions, Azure Pipelines - Windows: GitHub Actions, Azure Pipelines Queue @@ -59,7 +59,7 @@ queue for the tasks. Anyone can host a ``queue`` repository (usually named ``/crossbow``). A job is a git commit on a particular git branch, containing the required -configuration files to run the requested builds (like ``.travis.yml``, +configuration files to run the requested builds (like ``.travis.yml``, ``azure-pipelines.yml``, or ``crossbow.yml`` for `GitHub Actions`_ ). Scheduler @@ -118,7 +118,7 @@ to step 3: ``https://travis-ci.com///settings`` - Confirm the `auto cancellation`_ feature is turned off for branch builds. This should be the default setting. - + 7. Install Python (minimum supported version is 3.8): | Miniconda is preferred, see installation instructions: diff --git a/docs/source/developers/continuous_integration/overview.rst b/docs/source/developers/continuous_integration/overview.rst index 70323c9e48927..1d82e845a3360 100644 --- a/docs/source/developers/continuous_integration/overview.rst +++ b/docs/source/developers/continuous_integration/overview.rst @@ -26,7 +26,6 @@ Some files central to Arrow CI are: - ``docker-compose.yml`` - here we define docker services which can be configured using either enviroment variables, or the default values for these variables. - ``.env`` - here we define default values to configure the services in ``docker-compose.yml`` -- ``.travis.yml`` - here we define workflows which run on Travis - ``appveyor.yml`` - here we define workflows that run on Appveyor We use :ref:`Docker` in order to have portable and reproducible Linux builds, as well as running Windows builds in Windows containers. We use :ref:`Archery` and :ref:`Crossbow` to help co-ordinate the various CI tasks. @@ -60,7 +59,6 @@ The ``.yml`` files in ``.github/worflows`` are workflows which are run on GitHub There are two other files which define action-triggered builds: -- ``.travis.yml`` - runs on all commits and is used to test on architectures such as ARM and S390x - ``appveyor.yml`` - runs on commits related to Python or C++ Extended builds diff --git a/docs/source/developers/release.rst b/docs/source/developers/release.rst index cb0d713f50d0c..066400b33ffb5 100644 --- a/docs/source/developers/release.rst +++ b/docs/source/developers/release.rst @@ -497,8 +497,8 @@ Be sure to go through on the following checklist: Our CI systems give us some coverage for the things that CRAN checks, but there are a couple of final tests we should do to confirm that the release binaries will work and that everything runs on the same infrastructure that - CRAN has, which is difficult/impossible to emulate fully on Travis or with - Docker. For a precise list of checks, see the + CRAN has, which is difficult/impossible to emulate fully with Docker. For a + precise list of checks, see the `packaging checklist `_. Once all checks are clean, we submit to CRAN, which has a web form for