diff --git a/.github/workflows/branch42_scheduler.yml b/.github/workflows/branch42_scheduler.yml new file mode 100644 index 000000000000..08ea5ec2d7ed --- /dev/null +++ b/.github/workflows/branch42_scheduler.yml @@ -0,0 +1,114 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: "Branch-4.2 CI Scheduler" + +on: + schedule: + - cron: '0 3 */2 * *' + - cron: '0 6 */2 * *' + - cron: '0 8 */2 * *' + - cron: '0 13 */2 * *' + - cron: '0 15 */2 * *' + - cron: '0 17 */2 * *' + - cron: '0 18 */2 * *' + - cron: '0 20 */2 * *' + workflow_dispatch: + inputs: + target: + description: Target workflow to run + required: true + type: choice + default: all + options: + - all + - maven + - maven_java21 + - java17 + - java21 + - java25 + - non_ansi + - python_3.11 + - python_3.14 + +jobs: + schedule: + if: github.repository == 'apache/spark' + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + actions: write + steps: + - name: Maven Build + if: >- + github.event.schedule == '0 15 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'maven' || inputs.target == 'all')) + run: | + gh workflow run build_maven.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Maven Java 21 Build + if: >- + github.event.schedule == '0 17 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'maven_java21' || inputs.target == 'all')) + run: | + gh workflow run build_maven_java21.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Java 17 Build + if: >- + github.event.schedule == '0 13 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'java17' || inputs.target == 'all')) + run: | + gh workflow run build_java17.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Java 21 Build + if: >- + github.event.schedule == '0 6 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'java21' || inputs.target == 'all')) + run: | + gh workflow run build_java21.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Java 25 Build + if: >- + github.event.schedule == '0 8 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'java25' || inputs.target == 'all')) + run: | + gh workflow run build_java25.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Non-ANSI Build + if: >- + github.event.schedule == '0 3 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'non_ansi' || inputs.target == 'all')) + run: | + gh workflow run build_non_ansi.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Python 3.11 Build + if: >- + github.event.schedule == '0 18 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'python_3.11' || inputs.target == 'all')) + run: | + gh workflow run build_python_3.11.yml --repo ${{ github.repository }} --ref branch-4.2 + - name: Python 3.14 Build + if: >- + github.event.schedule == '0 20 */2 * *' || + (github.event_name == 'workflow_dispatch' && + (inputs.target == 'python_3.14' || inputs.target == 'all')) + run: | + gh workflow run build_python_3.14.yml --repo ${{ github.repository }} --ref branch-4.2 diff --git a/.github/workflows/build_branch42.yml b/.github/workflows/build_branch42.yml deleted file mode 100644 index 159a213c837b..000000000000 --- a/.github/workflows/build_branch42.yml +++ /dev/null @@ -1,52 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build (branch-4.2, Scala 2.13, JDK 17)" - -on: - schedule: - - cron: '0 13 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 17 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "SCALA_PROFILE": "scala2.13", - "PYSPARK_IMAGE_TO_TEST": "", - "PYTHON_TO_TEST": "" - } - jobs: >- - { - "build": "true", - "sparkr": "true", - "tpcds-1g": "true", - "docker-integration-tests": "true", - "k8s-integration-tests": "true", - "lint" : "true" - } diff --git a/.github/workflows/build_branch42_java21.yml b/.github/workflows/build_branch42_java21.yml deleted file mode 100644 index 59f10d247b7b..000000000000 --- a/.github/workflows/build_branch42_java21.yml +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build (branch-4.2, Scala 2.13, JDK 21)" - -on: - schedule: - - cron: '0 6 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 21 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-311", - "PYTHON_TO_TEST": "python3.11", - "SKIP_MIMA": "true", - "SKIP_UNIDOC": "true", - "DEDICATED_JVM_SBT_TESTS": "org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV1Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV2Suite" - } - jobs: >- - { - "build": "true", - "pyspark": "true", - "sparkr": "true", - "tpcds-1g": "true", - "docker-integration-tests": "true", - "yarn": "true", - "k8s-integration-tests": "true", - "buf": "true", - "ui": "true" - } diff --git a/.github/workflows/build_branch42_java25.yml b/.github/workflows/build_branch42_java25.yml deleted file mode 100644 index 4ce647c47605..000000000000 --- a/.github/workflows/build_branch42_java25.yml +++ /dev/null @@ -1,57 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build (branch-4.2, Scala 2.13, JDK 25)" - -on: - schedule: - - cron: '0 8 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 25 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-311", - "PYTHON_TO_TEST": "python3.11", - "SKIP_MIMA": "true", - "SKIP_UNIDOC": "true", - "DEDICATED_JVM_SBT_TESTS": "org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV1Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV2Suite" - } - jobs: >- - { - "build": "true", - "pyspark": "true", - "sparkr": "true", - "tpcds-1g": "true", - "docker-integration-tests": "true", - "yarn": "true", - "k8s-integration-tests": "true", - "buf": "true", - "ui": "true" - } diff --git a/.github/workflows/build_branch42_maven.yml b/.github/workflows/build_branch42_maven.yml deleted file mode 100644 index 3bf12f131aa5..000000000000 --- a/.github/workflows/build_branch42_maven.yml +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Maven (branch-4.2, Scala 2.13, JDK 17)" - -on: - schedule: - - cron: '0 15 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/maven_test.yml - if: github.repository == 'apache/spark' - with: - branch: branch-4.2 diff --git a/.github/workflows/build_branch42_maven_java21.yml b/.github/workflows/build_branch42_maven_java21.yml deleted file mode 100644 index 85635b1d681c..000000000000 --- a/.github/workflows/build_branch42_maven_java21.yml +++ /dev/null @@ -1,36 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Maven (branch-4.2, Scala 2.13, JDK 21)" - -on: - schedule: - - cron: '0 17 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/maven_test.yml - if: github.repository == 'apache/spark' - with: - branch: branch-4.2 - java: 21 diff --git a/.github/workflows/build_branch42_non_ansi.yml b/.github/workflows/build_branch42_non_ansi.yml deleted file mode 100644 index 4f9579cd012f..000000000000 --- a/.github/workflows/build_branch42_non_ansi.yml +++ /dev/null @@ -1,54 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Non-ANSI (branch-4.2, JDK 17, Scala 2.13)" - -on: - schedule: - - cron: '0 3 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 17 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-311", - "PYTHON_TO_TEST": "python3.11", - "SPARK_ANSI_SQL_MODE": "false", - } - jobs: >- - { - "build": "true", - "build-core-utils": "false", - "docs": "true", - "pyspark": "true", - "sparkr": "true", - "tpcds-1g": "true", - "docker-integration-tests": "true", - "yarn": "true" - } diff --git a/.github/workflows/build_branch42_python.yml b/.github/workflows/build_branch42_python.yml deleted file mode 100644 index c205e1e1c217..000000000000 --- a/.github/workflows/build_branch42_python.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Python-only (branch-4.2)" - -on: - schedule: - - cron: '0 18 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 17 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-311", - "PYTHON_TO_TEST": "python3.11" - } - jobs: >- - { - "pyspark": "true", - "pyspark-pandas": "true" - } diff --git a/.github/workflows/build_branch42_python_3.14.yml b/.github/workflows/build_branch42_python_3.14.yml deleted file mode 100644 index 92858065d18e..000000000000 --- a/.github/workflows/build_branch42_python_3.14.yml +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -name: "Build / Python-only (branch-4.2, Python 3.14)" - -on: - schedule: - - cron: '0 20 */2 * *' - workflow_dispatch: - -jobs: - run-build: - permissions: - packages: write - name: Run - uses: ./.github/workflows/build_and_test.yml - if: github.repository == 'apache/spark' - with: - java: 17 - branch: branch-4.2 - hadoop: hadoop3 - envs: >- - { - "PYSPARK_IMAGE_TO_TEST": "python-314", - "PYTHON_TO_TEST": "python3.14" - } - jobs: >- - { - "pyspark": "true", - "pyspark-pandas": "true" - } diff --git a/README.md b/README.md index d98785cdc793..717822ef4203 100644 --- a/README.md +++ b/README.md @@ -62,14 +62,15 @@ This README file only contains basic setup instructions. | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21.yml/badge.svg?branch=branch-4.x)](https://github.com/apache/spark/actions/workflows/build_maven_java21.yml?query=branch%3Abranch-4.x) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg?branch=branch-4.x)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml?query=branch%3Abranch-4.x) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml/badge.svg?branch=branch-4.x)](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml?query=branch%3Abranch-4.x) | -| branch-4.2 | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_java21.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_java21.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_java25.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_java25.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_non_ansi.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_non_ansi.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_maven.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_maven.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_maven_java21.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_maven_java21.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_python.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_python.yml) | -| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_branch42_python_3.14.yml/badge.svg)](https://github.com/apache/spark/actions/workflows/build_branch42_python_3.14.yml) | +| branch-4.2 | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_main.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_main.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_java17.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_java17.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_java21.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_java21.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_java25.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_java25.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_non_ansi.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_non_ansi.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_maven.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_maven_java21.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_maven_java21.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_python_3.11.yml?query=branch%3Abranch-4.2) | +| | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml/badge.svg?branch=branch-4.2)](https://github.com/apache/spark/actions/workflows/build_python_3.14.yml?query=branch%3Abranch-4.2) | | branch-4.1 | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_main.yml/badge.svg?branch=branch-4.1)](https://github.com/apache/spark/actions/workflows/build_main.yml?query=branch%3Abranch-4.1) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_java17.yml/badge.svg?branch=branch-4.1)](https://github.com/apache/spark/actions/workflows/build_java17.yml?query=branch%3Abranch-4.1) | | | [![GitHub Actions Build](https://github.com/apache/spark/actions/workflows/build_java21.yml/badge.svg?branch=branch-4.1)](https://github.com/apache/spark/actions/workflows/build_java21.yml?query=branch%3Abranch-4.1) |