Build against arrow-ballista in CI & remove ballista code from this repo

.github/pull_request_template.md

@@ -25,3 +25,23 @@ If there are user-facing changes then we may require documentation to be updated
 <!--
 If there are any breaking changes to public APIs, please add the `api change` label.
 -->
+
+# Does this PR break compatibility with Ballista?
+
+<!--
+The CI checks will attempt to build [arrow-ballista](https://github.com/apache/arrow-ballista) against this PR. If 
+this check fails then it indicates that this PR makes a breaking change to the DataFusion API.
+
+If possible, try to make the change in a way that is not a breaking API change. For example, if code has moved 
+ around, try adding `pub use` from the original location to preserve the current API.
+
+If it is not possible to avoid a breaking change (such as when adding enum variants) then follow this process:
+
+- Make a corresponding PR against `arrow-ballista` with the changes required there
+- Update `dev/build-arrow-ballista.sh` to clone the appropriate `arrow-ballista` repo & branch
+- Merge this PR when CI passes
+- Merge the Ballista PR
+- Create a new PR here to reset `dev/build-arrow-ballista.sh` to point to `arrow-ballista` master again
+
+_If you would like to help improve this process, please see https://github.com/apache/arrow-datafusion/issues/2583_
+-->

.github/workflows/dev.yml

@@ -66,9 +66,8 @@ jobs:
           #
           # ignore subproject CHANGELOG.md because they are machine generated
           npx prettier@2.3.2 --write \
-            '{ballista,datafusion,datafusion-examples,docs,python}/**/*.md' \
-            '!{ballista,datafusion,python}/CHANGELOG.md' \
+            '{datafusion,datafusion-examples,docs,python}/**/*.md' \
+            '!{datafusion,python}/CHANGELOG.md' \
             README.md \
-            CONTRIBUTING.md \
-            'ballista/**/*.{ts,tsx}'
+            CONTRIBUTING.md
           git diff --exit-code

.github/workflows/dev_pr/labeler.yml

@@ -20,10 +20,6 @@ datafusion:
   - datafusion-cli/**/*
   - datafusion-examples/**/*
 
-ballista:
-  - ballista/**/*
-  - ballista-examples/**/*
-
 python:
   - python/**/*
 
@@ -41,5 +37,4 @@ documentation:
   - README.md
   - ./**/README.md
   - DEVELOPERS.md
-  - ballista/docs/**.*
   - datafusion/docs/**.*

.github/workflows/rust.yml

@@ -121,8 +121,6 @@ jobs:
           cargo run --example csv_sql
           cargo run --example parquet_sql
           cargo run --example avro_sql --features=datafusion/avro
-          cd ../ballista-examples
-          cargo run --example test_sql --features=ballista/standalone
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -146,6 +144,9 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: true
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.x'
       - name: Cache Cargo
         uses: actions/cache@v2
         with:
@@ -162,16 +163,9 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ matrix.rust }}
-      # Ballista is currently not part of the main workspace so requires a separate test step
-      - name: Run Ballista tests
+      - name: Run tests
         run: |
-          export ARROW_TEST_DATA=$(pwd)/testing/data
-          export PARQUET_TEST_DATA=$(pwd)/parquet-testing/data
-          cd ballista/rust
-          # snmalloc requires cmake so build without default features
-          cargo test --no-default-features --features sled
-          # Ensure also compiles in standalone mode
-          cargo test --no-default-features --features standalone
+          ./dev/build-arrow-ballista.sh
         env:
           CARGO_HOME: "/github/home/.cargo"
           CARGO_TARGET_DIR: "/github/home/target"
@@ -237,8 +231,6 @@ jobs:
           POSTGRES_PASSWORD: postgres
       - name: Build datafusion-cli
         run: (cd datafusion-cli && cargo build)
-      - name: Build ballista-cli
-        run: (cd ballista-cli && cargo build)
       - name: Test Psql Parity
         run: python -m pytest -v integration-tests/test_psql_parity.py
         env:

.gitignore

@@ -96,3 +96,6 @@ venv/*
 
 # apache release artifacts
 dev/dist
+
+# CI
+arrow-ballista

Cargo.toml

@@ -27,13 +27,8 @@ members = [
     "datafusion/row",
     "datafusion-examples",
     "benchmarks",
-    "ballista/rust/client",
-    "ballista/rust/core",
-    "ballista/rust/executor",
-    "ballista/rust/scheduler",
-    "ballista-examples",
 ]
-exclude = ["ballista-cli", "datafusion-cli"]
+exclude = ["datafusion-cli"]
 
 [profile.release]
 codegen-units = 1