From d7fa6bce69aa043f127e5a78292ede20714d2785 Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Wed, 30 Jun 2021 17:18:27 +0800 Subject: [PATCH] bring back dev scripts for ballista --- dev/build-ballista-docker-arm64.sh | 34 +++++++++ dev/build-ballista-docker.sh | 24 +++++++ dev/build-set-env.sh | 20 ++++++ dev/build-ui.sh | 23 ++++++ dev/release/rat_exclude_files.txt | 108 +++++++++++++++++++++++++++++ dev/update_arrow_deps.py | 83 ++++++++++++++++++++++ 6 files changed, 292 insertions(+) create mode 100755 dev/build-ballista-docker-arm64.sh create mode 100755 dev/build-ballista-docker.sh create mode 100755 dev/build-set-env.sh create mode 100755 dev/build-ui.sh create mode 100644 dev/release/rat_exclude_files.txt create mode 100755 dev/update_arrow_deps.py diff --git a/dev/build-ballista-docker-arm64.sh b/dev/build-ballista-docker-arm64.sh new file mode 100755 index 000000000000..5d951773ada4 --- /dev/null +++ b/dev/build-ballista-docker-arm64.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ -z "${DOCKER_REPO}" ]; then + echo "DOCKER_REPO env var must be set" + exit -1 +fi +cargo install cross +cross build --release --target aarch64-unknown-linux-gnu +rm -rf temp-ballista-docker +mkdir temp-ballista-docker +cp target/aarch64-unknown-linux-gnu/release/ballista-executor temp-ballista-docker +cp target/aarch64-unknown-linux-gnu/release/ballista-scheduler temp-ballista-docker +cp target/aarch64-unknown-linux-gnu/release/tpch temp-ballista-docker +mkdir temp-ballista-docker/queries/ +cp benchmarks/queries/*.sql temp-ballista-docker/queries/ +docker buildx build --push -t $DOCKER_REPO/ballista-arm64 --platform=linux/arm64 -f dev/docker/ballista-arm64.Dockerfile temp-ballista-docker +rm -rf temp-ballista-docker \ No newline at end of file diff --git a/dev/build-ballista-docker.sh b/dev/build-ballista-docker.sh new file mode 100755 index 000000000000..bc028da9e716 --- /dev/null +++ b/dev/build-ballista-docker.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +. ./dev/build-set-env.sh +docker build -t ballista-base:$BALLISTA_VERSION -f dev/docker/ballista-base.dockerfile . +docker build -t ballista:$BALLISTA_VERSION -f dev/docker/ballista.dockerfile . diff --git a/dev/build-set-env.sh b/dev/build-set-env.sh new file mode 100755 index 000000000000..3eb29e7ce144 --- /dev/null +++ b/dev/build-set-env.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +export BALLISTA_VERSION=$(awk -F'[ ="]+' '$1 == "version" { print $2 }' ballista/rust/core/Cargo.toml) diff --git a/dev/build-ui.sh b/dev/build-ui.sh new file mode 100755 index 000000000000..070839702500 --- /dev/null +++ b/dev/build-ui.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +. ./dev/build-set-env.sh +docker build -t ballista-scheduler-ui:$BALLISTA_VERSION -f dev/docker/ballista-scheduler-ui.dockerfile ballista/ui/scheduler diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt new file mode 100644 index 000000000000..96beccd0af81 --- /dev/null +++ b/dev/release/rat_exclude_files.txt @@ -0,0 +1,108 @@ +*.npmrc +*.gitignore +*.dockerignore +.gitmodules +*_generated.h +*_generated.js +*_generated.ts +*.csv +*.json +*.snap +.github/ISSUE_TEMPLATE/*.md +.github/pull_request_template.md +ci/etc/rprofile +ci/etc/*.patch +ci/vcpkg/*.patch +CHANGELOG.md +dev/requirements*.txt +dev/archery/MANIFEST.in +dev/archery/requirements*.txt +dev/archery/archery/tests/fixtures/* +dev/archery/archery/crossbow/tests/fixtures/* +dev/release/rat_exclude_files.txt +dev/tasks/homebrew-formulae/apache-arrow.rb +dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install +dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat +dev/tasks/linux-packages/apache-arrow-apt-source/debian/control +dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules +dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format +dev/tasks/linux-packages/apache-arrow/debian/compat +dev/tasks/linux-packages/apache-arrow/debian/control.in +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow-python400.install +dev/tasks/linux-packages/apache-arrow/debian/libarrow400.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libgandiva400.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libparquet400.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links +dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib400.install +dev/tasks/linux-packages/apache-arrow/debian/libplasma400.install +dev/tasks/linux-packages/apache-arrow/debian/patches/series +dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install +dev/tasks/linux-packages/apache-arrow/debian/rules +dev/tasks/linux-packages/apache-arrow/debian/source/format +dev/tasks/linux-packages/apache-arrow/debian/watch +dev/tasks/requirements*.txt +dev/tasks/conda-recipes/* +pax_global_header +MANIFEST.in +__init__.pxd +__init__.py +requirements.txt +*.html +*.sgml +*.css +*.png +*.ico +*.svg +*.devhelp2 +*.scss +.gitattributes +rust-toolchain +benchmarks/queries/q*.sql +ballista/rust/scheduler/testdata/* +ballista/ui/scheduler/yarn.lock +python/rust-toolchain +python/requirements*.txt diff --git a/dev/update_arrow_deps.py b/dev/update_arrow_deps.py new file mode 100755 index 000000000000..44bdf4235d1c --- /dev/null +++ b/dev/update_arrow_deps.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Script that updates the arrow dependencies in datafusion and ballista, locall +# +# installation: +# pip install tomlkit requests +# +# usage: +# python update_arrow_deps.py + +from pathlib import Path + +# use tomlkit as it preserves comments and other formatting +import tomlkit +import requests + + +# find latest arrow-rs sha +def get_arrow_sha(): + url = 'https://api.github.com/repos/apache/arrow-rs/branches/master' + response = requests.get(url) + return response.json()['commit']['sha'] + + +# Update all entries that look like +# { +# 'git': 'https://github.com/apache/arrow-rs', +# 'rev': 'c3fe3bab9905739fdda75301dab07a18c91731bd' +# } +# to point at a new SHA +def update_dependencies(dependencies, new_sha): + if dependencies is None: + return + for dep_name in dependencies: + dep = dependencies[dep_name] + if hasattr(dep, 'get'): + if dep.get('git') == 'https://github.com/apache/arrow-rs': + dep['rev'] = new_sha + + +def update_cargo_toml(cargo_toml, new_sha): + print('updating {}'.format(cargo_toml.absolute())) + with open(cargo_toml) as f: + data = f.read() + + doc = tomlkit.parse(data) + + update_dependencies(doc.get('dependencies'), new_sha) + update_dependencies(doc.get('dev-dependencies'), new_sha) + + with open(cargo_toml, 'w') as f: + f.write(tomlkit.dumps(doc)) + + +# Begin main script + +repo_root = Path(__file__).parent.parent.absolute() + + +new_sha = get_arrow_sha() + +print('Updating files in {} to use sha {}'.format(repo_root, new_sha)) + + +for cargo_toml in repo_root.rglob('Cargo.toml'): + update_cargo_toml(cargo_toml, new_sha)