From d67bf4c7911fef4f30dc62756cc36ffcbeb87d10 Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Wed, 29 Oct 2025 12:12:18 +0800 Subject: [PATCH] CI: migrate from Jenkins to GitHub Actions * Move CI/CD pipeline from Jenkins to GitHub Actions * Split workflows into two separate files: - license-check.yml: Apache RAT and binary compliance checks - build-test.yml: MADlib build and test process * Preserve existing PostgreSQL configurations and test setup * Maintain compatibility with current test exclusion rules * Keep tool scripts in original locations for reference * Rename Jenkinsfile to Jenkinsfile.deprecated This migration enables: - Better integration with GitHub ecosystem - Improved visibility of CI/CD processes - Parallel execution of license checks and build tests - Support trigger the CI by push event or manual --- .github/workflows/build-test.yml | 150 ++++++++++++++++++++++++++ .github/workflows/license-check.yml | 117 ++++++++++++++++++++ Jenkinsfile => Jenkinsfile.deprecated | 0 pom.xml | 4 +- 4 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/build-test.yml create mode 100644 .github/workflows/license-check.yml rename Jenkinsfile => Jenkinsfile.deprecated (100%) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 000000000..9604240e3 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,150 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: MADlib CI Checks + +on: + push: + branches: [ madlib2-master, github-actions ] + pull_request: + branches: [ madlib2-master, github-actions ] + # Allow manual triggers for testing + workflow_dispatch: + +permissions: + contents: read + actions: read + checks: write + +jobs: + build-and-test: + name: Build and Test MADlib + runs-on: ubuntu-latest + + container: + image: madlib/postgres_15:jenkins + options: >- + --ulimit core=-1 + --privileged + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + set-safe-directory: true + + - name: Configure PostgreSQL + run: | + cp tool/pg_hba.conf.postgres /etc/postgresql/15/main/pg_hba.conf + echo " * soft nproc unlimited" > /etc/security/limits.d/postgres-limits.conf + service postgresql start + sleep 5 + # Verify PostgreSQL started successfully and show version + echo "PostgreSQL version:" && su -s /bin/bash - postgres -c 'psql -c "SELECT version()"' || exit 1 + + - name: Install Dependencies + run: | + # Fix PostgreSQL APT repository issue (focal-pgdg is no longer available) + rm -f /etc/apt/sources.list.d/pgdg.list || true + + apt-get update + apt-get install -y python3-pip openjdk-11-jre-headless + # Install Python packages (pypmml requires Java to be installed first) + pip install mock pandas numpy xgboost scikit-learn pyyaml pyxb-x pypmml + + - name: Build MADlib + run: | + mkdir -p logs + rm -rf build && mkdir -p build + cd build + echo "---------- CMake Configuration -----------" + cmake .. 2>&1 | tee ../logs/madlib_compile.log + echo "---------- Make Clean -----------" + make clean 2>&1 | tee -a ../logs/madlib_compile.log + echo "---------- Make Build (first pass) -----------" + make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log + echo "---------- Make Build (second pass) -----------" + make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log + echo "---------- Make Install -----------" + make install 2>&1 | tee -a ../logs/madlib_compile.log + echo "---------- Make Package -----------" + make package 2>&1 | tee -a ../logs/madlib_compile.log + chown -R postgres:postgres . || true + + - name: Run Tests + run: | + export PATH=$PATH:/usr/lib/postgresql/15/bin/ + WORKSPACE=$(pwd) + + echo "---------- Installing MADlib -----------" + # Install MADlib as postgres user + # Note: Use absolute path because 'su -' resets working directory + su -s /bin/bash - postgres -c "export PATH=\$PATH:/usr/lib/postgresql/15/bin/; cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres install" 2>&1 | tee $WORKSPACE/logs/madlib_install.log + + mkdir -p /tmp + + echo "---------- Removing known problematic test files -----------" + # Remove known problematic test files from BUILD directory (as done in jenkins_build.sh) + # These tests are known to fail in the Docker environment + rm -rf $WORKSPACE/build/src/ports/postgres/modules/deep_learning/test || true + rm -rf $WORKSPACE/build/src/ports/postgres/15/modules/deep_learning/test || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/linalg/test/linalg.sql_in || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/prob/test/prob.sql_in || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/stats/test/cox_prop_hazards.sql_in || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/utilities/test/path.sql_in || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/mxgboost/test/madlib_xgboost.sql_in || true + rm -rf $WORKSPACE/build/src/ports/postgres/modules/kmeans/test/kmeans.sql_in || true + + echo "---------- Running dev-check -----------" + # Run dev-check and unit tests as postgres user + # Note: These commands will fail the workflow if tests fail (no || true) + su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check" 2>&1 | tee $WORKSPACE/logs/madlib_dev_check.log + + echo "---------- Running unit tests -----------" + su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp unit-test" 2>&1 | tee -a $WORKSPACE/logs/madlib_dev_check.log + + - name: Process Test Results + if: always() + run: | + echo "---------- Converting test results to JUnit format -----------" + WORKSPACE=$(pwd) + python3 tool/jenkins/junit_export.py . $WORKSPACE/logs/madlib_dev_check.log $WORKSPACE/logs/madlib_dev_check.xml + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + files: '${{ github.workspace }}/logs/madlib_dev_check.xml' + check_name: '๐Ÿงช MADlib Test Results' + comment_mode: 'always' + + - name: Upload Build Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: madlib-packages + path: build/*.deb + if-no-files-found: warn + + - name: Upload Build Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: madlib-build-logs + path: logs/ + if-no-files-found: warn \ No newline at end of file diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml new file mode 100644 index 000000000..534075832 --- /dev/null +++ b/.github/workflows/license-check.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: License Check + +on: + push: + branches: [ madlib2-master, github-actions ] + pull_request: + branches: [ madlib2-master, github-actions ] + # Allow manual triggers for compliance verification + workflow_dispatch: + +jobs: + rat-check: + name: Apache RAT License Check + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Set up JDK and Maven + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + cache: 'maven' + + - name: Run Apache RAT Check + run: | + echo "Running Apache Rat license check..." + mvn apache-rat:check | tee rat-output.log + + if grep -q "\[INFO\] BUILD FAILURE" rat-output.log; then + echo "::error::Apache Rat check failed - build failure detected" + exit 1 + fi + + - name: Run NOTICE Year and binary file checks + run: | + set -o pipefail + chmod +x ./tool/jenkins/rat_check.sh + ./tool/jenkins/rat_check.sh 2>&1 | tee license_checks.txt + + - name: Upload Rat check results + if: always() + uses: actions/upload-artifact@v4 + with: + name: rat-check-results + path: | + rat-output.log + license_checks.txt + retention-days: 7 + + - name: Generate Summary + if: always() + run: | + # Check RAT results + if [[ -f rat-output.log ]] && grep -q "\[INFO\] BUILD SUCCESS" rat-output.log; then + echo "### โœ… License Check Passed" >> "$GITHUB_STEP_SUMMARY" + echo "All files comply with Apache License requirements." >> "$GITHUB_STEP_SUMMARY" + elif [[ -f rat-output.log ]]; then + echo "### โŒ License Check Failed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + + if grep -q "Files with unapproved licenses:" rat-output.log; then + echo "**Files with unapproved licenses:**" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + sed -n '/Files with unapproved licenses:/,/\[INFO\] ------------------------------------------------------------------------/p' rat-output.log | \ + grep -v "\[INFO\] ------------------------------------------------------------------------" | \ + grep -v "^$" | \ + head -20 | \ + sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + fi + + if grep -q "Rat check: Summary over all files" rat-output.log; then + echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY" + grep "Rat check: Summary over all files" rat-output.log | sed 's/\[INFO\] //' >> "$GITHUB_STEP_SUMMARY" + fi + else + echo "### โš ๏ธ No RAT Output Log Found" >> "$GITHUB_STEP_SUMMARY" + fi + + # Check additional checks results + echo "" >> "$GITHUB_STEP_SUMMARY" + if [[ -f license_checks.txt ]]; then + if grep -q "Error" license_checks.txt || grep -q "FAILED" license_checks.txt; then + echo "### โŒ Additional Checks Failed" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Check output:**" >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + cat license_checks.txt >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + else + echo "### โœ… Additional Checks Passed" >> "$GITHUB_STEP_SUMMARY" + echo "- NOTICE file year is current" >> "$GITHUB_STEP_SUMMARY" + echo "- Version numbers match" >> "$GITHUB_STEP_SUMMARY" + echo "- No unexpected binary files found" >> "$GITHUB_STEP_SUMMARY" + fi + fi \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile.deprecated similarity index 100% rename from Jenkinsfile rename to Jenkinsfile.deprecated diff --git a/pom.xml b/pom.xml index f70c25d50..c29eac547 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ org.apache.rat apache-rat-plugin - 0.11 + 0.16.1 @@ -692,7 +692,7 @@ org.apache.rat apache-rat-plugin - 0.11 + 0.16.1 verify