From 4746ac80e9e0a4f17215e064c9c05e248688fb87 Mon Sep 17 00:00:00 2001 From: Evert Lammerts Date: Wed, 5 Nov 2025 15:03:51 +0100 Subject: [PATCH] Create a PR to bump submodule if nightly is stale --- .github/workflows/release.yml | 61 ++++++++--- .github/workflows/submodule_auto_pr.yml | 128 ++++++++++++++++++++++++ 2 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/submodule_auto_pr.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f3550fb0..727f8027 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,11 @@ on: options: - test - prod + nightly-stale-after-days: + type: string + description: After how many days should nightlies be considered stale + required: true + default: 3 store-s3: type: boolean description: Also store test packages in S3 (always true for prod) @@ -41,6 +46,17 @@ jobs: duckdb-sha: ${{ inputs.duckdb-sha }} set-version: ${{ inputs.stable-version }} + submodule_pr: + name: Create or update PR to bump submodule to given SHA + needs: build_sdist + uses: ./.github/workflows/submodule_auto_pr.yml + with: + duckdb-python-sha: ${{ inputs.duckdb-python-sha }} + duckdb-sha: ${{ inputs.duckdb-sha }} + secrets: + # reusable workflows and secrets are not great: https://github.com/actions/runner/issues/3206 + DUCKDBLABS_BOT_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + workflow_state: name: Set state for the release workflow needs: build_sdist @@ -51,23 +67,36 @@ jobs: runs-on: ubuntu-latest steps: - id: index_check - name: Check ${{ needs.build_sdist.outputs.package-version }} on PyPI + name: Check version on PyPI run: | - set -eu - # Check PyPI whether the release we're building is already present + set -ex pypi_hostname=${{ inputs.pypi-index == 'test' && 'test.' || '' }}pypi.org - pkg_version=${{ needs.build_sdist.outputs.package-version }} - url=https://${pypi_hostname}/pypi/duckdb/${pkg_version}/json - http_status=$( curl -s -o /dev/null -w "%{http_code}" $url || echo $? ) - if [[ $http_status == "200" ]]; then - echo "::warning::Package version ${pkg_version} is already present on ${pypi_hostname}" - pypi_state=VERSION_FOUND - elif [[ $http_status == 000* ]]; then - echo "::error::Error checking PyPI at ${url}: curl exit code ${http_status#'000'}" - pypi_state=UNKNOWN - else - echo "::notice::Package version ${pkg_version} not found on ${pypi_hostname} (http status: ${http_status})" + # install duckdb + curl https://install.duckdb.org | sh + # query pypi + result=$(cat <>'upload_time_iso_8601')::DATE AS age, + FROM read_json('https://${pypi_hostname}/pypi/duckdb/json') AS jd + CROSS JOIN json_each(jd.releases) AS rel(key, value) + CROSS JOIN unnest(FROM_JSON(rel.value, '["JSON"]')) AS file(value) + WHERE rel.key='${{ needs.build_sdist.outputs.package-version }}' + LIMIT 1; + EOF + ) + if [ -z "$result" ]; then pypi_state=VERSION_NOT_FOUND + else + pypi_state=VERSION_FOUND + fi + if [[ -z "${{ inputs.stable-version }}" ]]; then + age=${result#age = } + if [ "${age}" -ge "${{ inputs.nightly-stale-after-days }}" ]; then + echo "::warning title=Stale nightly for ${{ github.ref_name }}::Nightly is ${age} days old (max=${{ inputs.nightly-stale-after-days }})" + fi fi echo "pypi_state=${pypi_state}" >> $GITHUB_OUTPUT @@ -96,7 +125,7 @@ jobs: echo "::notice::S3 upload disabled in inputs, not generating S3 URL" exit 0 fi - if [[ VERSION_FOUND == "${{ steps.index_check.outputs.pypi_state }}" ]]; then + if [[ VERSION_NOT_FOUND != "${{ steps.index_check.outputs.pypi_state }}" ]]; then echo "::warning::S3 upload disabled because package version already uploaded to PyPI" exit 0 fi @@ -110,7 +139,7 @@ jobs: build_wheels: name: Build and test releases needs: workflow_state - if: ${{ needs.workflow_state.outputs.pypi_state != 'VERSION_FOUND' }} + if: ${{ needs.workflow_state.outputs.pypi_state == 'VERSION_NOT_FOUND' }} uses: ./.github/workflows/packaging_wheels.yml with: minimal: false diff --git a/.github/workflows/submodule_auto_pr.yml b/.github/workflows/submodule_auto_pr.yml new file mode 100644 index 00000000..97c63ba0 --- /dev/null +++ b/.github/workflows/submodule_auto_pr.yml @@ -0,0 +1,128 @@ +name: Submodule Auto PR +on: + workflow_call: + inputs: + duckdb-python-sha: + type: string + description: The commit to build against (defaults to latest commit of current ref) + required: false + duckdb-sha: + type: string + description: The DuckDB submodule commit or ref to build against + required: true + auto-land: + type: boolean + description: Immediately merge the PR (placeholder - doesn't work) + default: false + secrets: + DUCKDBLABS_BOT_TOKEN: + description: Github token of the DuckDBLabs bot + required: true + +defaults: + run: + shell: bash + +jobs: + create_pr: + name: Create PR to bump duckdb submodule to given SHA + runs-on: ubuntu-latest + steps: + - name: Checkout DuckDB Python + uses: actions/checkout@v4 + with: + ref: ${{ inputs.duckdb-python-sha }} + fetch-depth: 0 + submodules: true + + - name: Checkout or Create Needed Branch + run: | + git fetch --all + head_sha=${{ inputs.duckdb-python-sha }} + branch_name="vendoring-${{ github.ref_name }}" + if [[ `git rev-parse --verify ${branch_name} 2>/dev/null` ]]; then + # branch exists + git checkout ${branch_name} + else + # new branch + git checkout -b ${branch_name} + fi + [[ ${head_sha} ]] && git reset --hard ${head_sha} || true + + - name: Checkout DuckDB at Given SHA + run: | + cd external/duckdb + git fetch origin + git checkout ${{ inputs.duckdb-sha }} + + - name: Determine GH PR Command + id: gh_pr_command + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + pr_url=$( gh pr list --head vendoring-${{ github.ref_name }} --state open --json url --jq '.[].url' ) + if [[ $pr_url ]]; then + echo "::notice::Found existing pr, will edit (${pr_url})" + gh_command="edit ${pr_url}" + else + echo "::notice::No existing PR, will create new" + gh_command="create --head vendoring-${{ github.ref_name }} --base ${{ github.ref_name }}" + fi + echo "subcommand=${gh_command}" >> $GITHUB_OUTPUT + + - name: Set Git User + run: | + git config --global user.email "github_bot@duckdblabs.com" + git config --global user.name "DuckDB Labs GitHub Bot" + + - name: Create PR to Bump DuckDB Submodule + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + # First commit and push + git add external/duckdb + git commit -m "Bump submodule" + git push --force origin vendoring-${{ github.ref_name }} + # create PR msg + echo "Bump duckdb submodule:" > body.txt + echo "- Target branch: ${{ github.ref_name }}" >> body.txt + echo "- Date: $( date +"%Y-%m-%d %H:%M:%S" )" >> body.txt + echo "- DuckDB SHA: ${{ inputs.duckdb-sha }}" >> body.txt + echo "- Trigger: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> body.txt + subcommand="${{ steps.gh_pr_command.outputs.subcommand }}" + gh pr ${subcommand} \ + --title "[duckdb-labs bot] Bump DuckDB submodule" \ + --body-file body.txt > output.txt 2>&1 + success=$? + # Show summary + url=$( [[ $success ]] && gh pr view vendoring-${{ github.ref_name }} --json url --jq .url || true ) + echo "## Submodule PR Summary" >> $GITHUB_STEP_SUMMARY + if [[ $success ]]; then + prefix=$( [[ $subcommand == edit* ]] && echo "Created" || echo "Updated" ) + echo "### ${prefix} PR: [${url}](${url})" >> $GITHUB_STEP_SUMMARY + else + echo "### Failed to create PR" >> $GITHUB_STEP_SUMMARY + fi + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + [[ $success ]] || exit 1 + + - name: Automerge PR + if: ${{ inputs.auto-land }} + env: + GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }} + run: | + # PLACEHOLDER: DUCKDBLABS_BOT_TOKEN DOES NOT HAVE PERMISSIONS TO MERGE PRS + set -ex + gh pr merge vendoring-${{ github.ref_name }} --rebase > output.txt + success=$? + # Show summary + if [[ $success ]]; then + echo "### PR merged" >> $GITHUB_STEP_SUMMARY + else + echo "### Failed to auto-merge PR" >> $GITHUB_STEP_SUMMARY + fi + echo '```' >> $GITHUB_STEP_SUMMARY + cat output.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY