Issue #65: Automatically convert new markdown files in bazel.build after submodule update

.github/workflows/dependabot-automerge.yml

@@ -13,27 +13,45 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  automerge:
-    timeout-minutes: 10
-    # Only run on Dependabot PRs targeting main
+  check-dependabot:
+    timeout-minutes: 5
     if: >
       github.event.pull_request.user.login == 'dependabot[bot]' &&
       github.event.pull_request.base.ref == 'main'
     runs-on: ubuntu-latest
+    outputs:
+      should_process: ${{ steps.check.outputs.should_process }}
     steps:
       - name: Fetch Dependabot metadata
         id: meta
         uses: dependabot/fetch-metadata@v2
         with:
-          # already added to repo
           github-token: ${{ secrets.GH_AUTOMERGE_PAT }}
 
-      # Gate: only submodules updates; only the "upstream" submodule path
+      - name: Check if upstream submodule
+        id: check
+        run: |
+          if [[ "${{ steps.meta.outputs.package-ecosystem }}" == "submodules" ]] && \
+             [[ "${{ steps.meta.outputs.dependency-names }}" == *"upstream"* ]]; then
+            echo "should_process=true" >> $GITHUB_OUTPUT
+          else
+            echo "should_process=false" >> $GITHUB_OUTPUT
+          fi
+
+  # Generate docs after dependabot updates the submodule
+  generate-docs:
+    needs: check-dependabot
+    if: needs.check-dependabot.outputs.should_process == 'true'
+    uses: ./.github/workflows/pull-from-bazel-build.yml
+    secrets: inherit
+
+  # Auto-merge after docs are generated
+  enable-automerge:
+    needs: [check-dependabot, generate-docs]
+    if: needs.check-dependabot.outputs.should_process == 'true'
+    runs-on: ubuntu-latest
+    steps:
       - name: Enable auto-merge (squash)
-        if: >
-          steps.meta.outputs.package-ecosystem == 'submodules' &&
-          contains(steps.meta.outputs.dependency-names, 'upstream')
         run: gh pr merge --auto --squash "${{ github.event.pull_request.html_url }}"
         env:
-          # already added to repo
-          GH_TOKEN: ${{ secrets.GH_AUTOMERGE_PAT }}
+          GH_TOKEN: ${{ secrets.GH_AUTOMERGE_PAT }}

.github/workflows/generate-docs.yml

@@ -0,0 +1,20 @@
+name: Generate Documentation on PR
+
+on:
+  pull_request:
+    types: [opened, reopened, synchronize, ready_for_review]
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: generate-docs-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  generate-docs:
+    # Skip Dependabot PRs (handled by auto-merge workflow)
+    if: github.event.pull_request.user.login != 'dependabot[bot]'
+    uses: ./.github/workflows/pull-from-bazel-build.yml
+    secrets: inherit

.github/workflows/pull-from-bazel-build.yml

@@ -1,107 +1,136 @@
-name: Pull Bazel Build Upstream Repo
+name: Pull Fresh Upstream Documentation
 
 on:
-  pull_request:
-    types: [opened, synchronize, reopened]
-  push:
-    branches: [main]
-    # called from trigger-from-bazel-repo.yml
-  workflow_dispatch:
-    inputs:
-        bazelCommitHash:
-          description: 'The commit hash of the Bazel repo to use'
-          type: string
-          default: origin/main
-  # allow debugging by triggering from the GitHub UI
   workflow_call:
     inputs:
-        bazelCommitHash:
-          description: 'The commit hash of the Bazel repo to use'
-          type: string
-          default: origin/main
+      bazelCommitHash:
+        description: 'Specific Bazel commit hash to checkout (optional)'
+        required: false
+        type: string
+        default: ''
+
 jobs:
   pull-fresh-upstream:
-    # don't run on dependabot PRs
-    if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     permissions:
-      contents: read
-      packages: write
+      contents: write
+      pull-requests: write
 
     steps:
-        - uses: actions/checkout@v5
-          with:
-            # Don't auto-init submodules
-            submodules: false
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          # Don't auto-init submodules
+          submodules: false
+          # This token is necessary so that the action can push into the repo
+          # and trigger the required check (this workflow). Otherwise, the 
+          # default GITHUB_TOKEN cannot be considered a 'synchronize' event
+          token: ${{ secrets.GH_AUTOMERGE_PAT }}
 
-        - name: Checkout submodules
-          run: git submodule update --init -- upstream
+      - name: Checkout submodules
+        run: git submodule update --init -- upstream
 
-        - name: Checkout commit of Bazel Build submodule
-          if: ${{ inputs.bazelCommitHash != '' }}
-          working-directory: upstream
-          run: git checkout '${{ inputs.bazelCommitHash }}'
+      - name: Checkout commit of Bazel Build submodule
+        if: ${{ inputs.bazelCommitHash != '' }}
+        working-directory: upstream
+        run: git checkout '${{ inputs.bazelCommitHash }}'
 
-        - name: Setup Bazel
-          uses: bazel-contrib/setup-bazel@0.15.0
-          with:
-            bazelisk-cache: true
-            repository-cache: true
+      - name: Setup Bazel
+        uses: bazel-contrib/setup-bazel@0.15.0
+        with:
+          bazelisk-cache: true
+          repository-cache: true
 
-        - name: Build reference documentation
-          working-directory: upstream
-          run: >
-            bazel build
-            --config=docs
-            --build_metadata=ROLE=DOCS
-            --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_ORG_API_KEY }}
-            --bes_results_url=https://app.buildbuddy.io/invocation/
-            --bes_backend=grpcs://remote.buildbuddy.io
-            --remote_cache=grpcs://remote.buildbuddy.io
-            --remote_timeout=10m
-            //src/main/java/com/google/devtools/build/lib:gen_reference_docs
+      - name: Build reference documentation
+        working-directory: upstream
+        run: >
+          bazel build
+          --config=docs
+          --build_metadata=ROLE=DOCS
+          --remote_header=x-buildbuddy-api-key=${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          --bes_results_url=https://app.buildbuddy.io/invocation/
+          --bes_backend=grpcs://remote.buildbuddy.io
+          --remote_cache=grpcs://remote.buildbuddy.io
+          --remote_timeout=10m
+          //src/main/java/com/google/devtools/build/lib:gen_reference_docs
 
-        # Upload reference-docs.zip as an artifact for debugging purposes
-        - name: Upload reference docs artifact
-          if: ${{ github.ref != 'refs/heads/main' }}
-          uses: actions/upload-artifact@v4.6.2
-          with:
-            name: reference-docs
-            path: upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip
-            retention-days: 7
+      - name: Upload reference docs artifact
+        if: ${{ github.ref != 'refs/heads/main' }}
+        uses: actions/upload-artifact@v4.6.2
+        with:
+          name: reference-docs
+          path: upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip
+          retention-days: 7
 
-        - name: Clean up mdx files
-          run: ./cleanup-mdx.sh
+      - name: Clean up mdx files
+        run: ./cleanup-mdx.sh
 
-        - name: Set up Go
-          uses: actions/setup-go@v6
-          with:
-            go-version: '1.25.2'
+      - name: Set up Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: '1.25.2'
 
-        - name: Initialize Go module for converter
-          run: |
-            cd html2md_converter
-            go mod init html-to-md-converter
-            go get github.com/JohannesKaufmann/html-to-markdown
+      - name: Initialize Go module for converter
+        run: |
+          cd html2md_converter
+          go mod init html-to-md-converter
+          go get github.com/JohannesKaufmann/html-to-markdown
 
-        - name: Build HTML to Markdown converter
-          run: |
-            cd html2md_converter
-            go build -o html-to-md main.go
+      - name: Build HTML to Markdown converter
+        run: |
+          cd html2md_converter
+          go build -o html-to-md main.go
 
-        - name: Convert reference documentation HTML to Markdown
-          run: |
-            # Extract and convert HTML reference docs to Markdown
-            ./html2md_converter/html-to-md \
-              -zip upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip \
-              -output reference-docs-temp
+      - name: Convert reference documentation HTML to Markdown
+        run: |
+          # Extract and convert HTML reference docs to Markdown
+          ./html2md_converter/html-to-md \
+            -zip upstream/bazel-bin/src/main/java/com/google/devtools/build/lib/reference-docs.zip \
+            -output reference-docs-temp
 
-        - name: Transform upstream docs to mdx
-          run: ./copy-upstream-docs.sh
+      - name: Transform upstream docs to mdx
+        run: ./copy-upstream-docs.sh
 
-        - name: Create versioned navigation
-          run: ./docs.json.update.sh
+      - name: Create versioned navigation
+        run: ./docs.json.update.sh
 
-        - name: Clean up temporary files
-          run: rm -rf reference-docs-temp
+      - name: Clean up temporary files
+        run: rm -rf reference-docs-temp
+
+      - name: Configure Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
 
+      - name: Commit and push changes
+        env:
+          BRANCH: ${{ github.head_ref || github.ref_name }}
+        run: |
+          set -euo pipefail
+
+          # Ensure local branch points at origin/BRANCH and is checked out
+          git fetch origin "$BRANCH"
+          git switch -C "$BRANCH" "origin/$BRANCH"
+          git branch --set-upstream-to="origin/$BRANCH" "$BRANCH"
+
+          # Rebase onto latest remote before creating a new commit
+          git pull
+
+          # Clean up any modified content in submodules (keep pointer changes)
+          # This handles the "modified content" issue while preserving submodule commit updates
+          cd upstream
+          git reset --hard
+          git clean -fd
+          cd ..
+
+          if [[ -n "$(git status --porcelain)" ]]; then
+            echo "Changes detected, committing and pushing..."
+            git add -A
+            git commit -m $'chore: update documentation from upstream Bazel repo\n\nGenerated by GitHub Actions workflow from upstream Bazel repository.\nThis commit includes transformed documentation files ready for Mintlify deployment.'
+
+            # Push back to the same branch
+            git push origin "HEAD:$BRANCH"
+            echo "Changes committed and pushed successfully"
+          else
+            echo "No changes detected, skipping commit"
+          fi

.gitignore

@@ -4,3 +4,7 @@ __pycache__/
 .venv/
 .hugo_build.lock
 node_modules
+# ignore go build files
+html2md_converter/go.sum
+html2md_converter/go.mod
+html2md_converter/html-to-md

basics/hermeticity.mdx

@@ -87,7 +87,7 @@ non-hermetic actions.
   rich enough to allow arbitrary processing to happen in the process. You can
   get a log of some potentially non-hermetic actions in Bazel workspace rules by
   adding the flag
-  `--experimental_workspace_rules_log_file={{ '<var>' }}PATH{{ '</var>' }}` to
+  `--experimental_workspace_rules_log_file=<var>PATH</var>` to
   your Bazel command.
 
 Note: Make your build fully hermetic when mixing remote and local execution,

cleanup-mdx.sh

@@ -12,8 +12,11 @@ LOCAL_FILES="
 
 echo "Cleaning up .mdx files..."
 
-# Find all .mdx files in the repo (excluding upstream, work, .github, .cursor directories)
-find . -name "*.mdx" -type f | while read -r mdx_file; do
+# Find all .mdx files in the repo, excluding versioned directories
+# This excludes any directory starting with a digit (e.g., 6.5.0, 7.6.1, 8.0.1)
+find . -name "*.mdx" -type f \
+  -not -path "./[0-9]*/*" \
+  -not -path "./.github/*" | while read -r mdx_file; do
 
     # Check if this exact file path is in LOCAL_FILES
     if echo "$LOCAL_FILES" | grep -q "^\s*$mdx_file\s*$"; then