Add Apache Airflow Provider Registry

.github/CODEOWNERS

@@ -107,6 +107,13 @@ airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/ @Lee-W @jason810496 @guan
 # Generated metadata
 /generated/provider_metadata.json @potiuk @eladkal @shahar1 @vincbeck @jscheffl
 
+# Provider Registry
+/registry/ @kaxil
+/dev/registry/ @kaxil
+/dev/breeze/src/airflow_breeze/commands/registry_commands*.py @kaxil
+/dev/breeze/src/airflow_breeze/utils/publish_registry_versions.py @kaxil
+/dev/breeze/tests/test_publish_registry_versions.py @kaxil
+
 # Dev tools
 /.github/workflows/ @potiuk @ashb @gopidesupavan @amoghrajesh @jscheffl @bugraoz93 @kaxil @jason810496
 /dev/ @potiuk @ashb @gopidesupavan @amoghrajesh @jscheffl @bugraoz93 @jason810496 @jedcunningham @ephraimbuddy

.github/boring-cyborg.yml

@@ -557,6 +557,14 @@ labelPRBasedOnFilePath:
   area:kubernetes-tests:
     - kubernetes-tests/**/*
 
+  area:registry:
+    - registry/**/*
+    - dev/registry/**/*
+    - dev/breeze/src/airflow_breeze/commands/registry_commands*.py
+    - dev/breeze/src/airflow_breeze/utils/publish_registry_versions.py
+    - dev/breeze/tests/test_publish_registry_versions.py
+    - .github/workflows/registry-*
+
 # Various Flags to control behaviour of the "Labeler"
 labelerFlags:
   # If this flag is changed to 'false', labels would only be added when the PR is first created

.github/workflows/publish-docs-to-s3.yml

@@ -93,6 +93,7 @@ jobs:
       # yamllint disable rule:line-length
       skip-write-to-stable-folder: ${{ inputs.skip-write-to-stable-folder && '--skip-write-to-stable-folder' || '' }}
       default-python-version: "3.10"
+      registry-providers: ${{ steps.parameters.outputs.registry-providers }}
     if: contains(fromJSON('[
       "ashb",
       "bugraoz93",
@@ -160,6 +161,19 @@ jobs:
              echo "airflow-version=no-airflow" >> ${GITHUB_OUTPUT}
              echo "airflow-base-version=no-airflow" >> ${GITHUB_OUTPUT}
           fi
+          # Extract provider IDs for registry update
+          # include-docs contains package names like "apache-airflow-providers-amazon"
+          # Strip the "apache-airflow-providers-" prefix to get provider IDs
+          REGISTRY_PROVIDERS=""
+          for pkg in ${INCLUDE_DOCS}; do
+            case "${pkg}" in
+              apache-airflow-providers-*)
+                PROVIDER_ID="${pkg#apache-airflow-providers-}"
+                REGISTRY_PROVIDERS="${REGISTRY_PROVIDERS:+${REGISTRY_PROVIDERS} }${PROVIDER_ID}"
+                ;;
+            esac
+          done
+          echo "registry-providers=${REGISTRY_PROVIDERS}" >> ${GITHUB_OUTPUT}
 
   build-docs:
     needs: [build-info]
@@ -304,7 +318,7 @@ jobs:
         with:
           name: airflow-docs
           path: /mnt/_build
-      - name: "Move doscs to generated folder"
+      - name: "Move docs to generated folder"
         run: mv /mnt/_build generated/_build
       - name: "Make sure SBOM dir exists and has the right permissions"
         run: |
@@ -377,3 +391,15 @@ jobs:
           breeze release-management publish-docs-to-s3 --source-dir-path ${SOURCE_DIR_PATH} \
           --destination-location ${DESTINATION_LOCATION} --stable-versions \
           --exclude-docs "${EXCLUDE_DOCS}" --overwrite ${SKIP_WRITE_TO_STABLE_FOLDER}
+
+  update-registry:
+    needs: [publish-docs-to-s3, build-info]
+    if: needs.build-info.outputs.registry-providers != ''
+    name: "Update Provider Registry"
+    uses: ./.github/workflows/registry-build.yml
+    with:
+      destination: ${{ needs.build-info.outputs.destination }}
+      provider: ${{ needs.build-info.outputs.registry-providers }}
+    secrets:
+      DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+      DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}

.github/workflows/registry-build.yml

@@ -0,0 +1,232 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Build & Publish Registry
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      destination:
+        description: "Publish to live or staging S3 bucket"
+        required: true
+        type: choice
+        options:
+          - staging
+          - live
+        default: staging
+      provider:
+        description: "Provider ID(s) for incremental build (space-separated, empty = full build)"
+        required: false
+        type: string
+        default: ""
+  workflow_call:
+    inputs:
+      destination:
+        description: "Publish to live or staging S3 bucket"
+        required: false
+        type: string
+        default: staging
+      provider:
+        description: "Provider ID(s) for incremental build (space-separated, empty = full build)"
+        required: false
+        type: string
+        default: ""
+    secrets:
+      DOCS_AWS_ACCESS_KEY_ID:
+        required: true
+      DOCS_AWS_SECRET_ACCESS_KEY:
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-and-publish-registry:
+    timeout-minutes: 30
+    name: "Build & Publish Registry"
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    if: >
+      github.event_name == 'workflow_call' ||
+      contains(fromJSON('[
+        "ashb",
+        "bugraoz93",
+        "eladkal",
+        "ephraimbuddy",
+        "jedcunningham",
+        "jscheffl",
+        "kaxil",
+        "pierrejeambrun",
+        "shahar1",
+        "potiuk",
+        "utkarsharma2",
+        "vincbeck"
+        ]'), github.event.sender.login)
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          persist-credentials: false
+
+      # --- Breeze setup ---
+      # All three extraction scripts run inside breeze so that
+      # extract_parameters.py and extract_connections.py can import provider
+      # classes at runtime.  extract_metadata.py also runs in breeze for
+      # consistency — it writes to dev/registry/ (mounted) so the other two
+      # scripts can read providers.json / modules.json from there.
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        with:
+          python-version: "3.12"
+
+      - name: "Build CI image"
+        # Fallback to raw docker buildx when breeze cache is stale — same
+        # pattern as publish-docs-to-s3.yml.
+        run: >
+          breeze ci-image build --python 3.12 ||
+          docker buildx build --load --builder default --progress=auto --pull
+          --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES=false
+          --build-arg AIRFLOW_USE_UV=true --build-arg BUILD_PROGRESS=auto
+          --build-arg INSTALL_MYSQL_CLIENT_TYPE=mariadb
+          --build-arg VERSION_SUFFIX_FOR_PYPI=dev0
+          -t ghcr.io/apache/airflow/main/ci/python3.12:latest --target main .
+          -f Dockerfile.ci --platform linux/amd64
+
+      - name: "Install AWS CLI v2"
+        run: |
+          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
+          unzip -q /tmp/awscliv2.zip -d /tmp
+          rm /tmp/awscliv2.zip
+          sudo /tmp/aws/install --update
+          rm -rf /tmp/aws/
+
+      - name: "Configure AWS credentials"
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a  # v4.0.1
+        with:
+          aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+
+      - name: "Determine S3 destination"
+        id: destination
+        env:
+          DESTINATION: ${{ inputs.destination || 'staging' }}
+        run: |
+          if [[ "${DESTINATION}" == "live" ]]; then
+            echo "bucket=s3://live-docs-airflow-apache-org/registry/" >> "${GITHUB_OUTPUT}"
+          else
+            echo "bucket=s3://staging-docs-airflow-apache-org/registry/" >> "${GITHUB_OUTPUT}"
+          fi
+          echo "Publishing registry to ${DESTINATION}"
+
+      # --- Incremental: download existing data from S3 ---
+      - name: "Download existing registry data from S3"
+        id: download-existing
+        if: inputs.provider != ''
+        env:
+          S3_BUCKET: ${{ steps.destination.outputs.bucket }}
+        run: |
+          mkdir -p /tmp/existing-registry
+          PROVIDERS_URL="${S3_BUCKET}api/providers.json"
+          MODULES_URL="${S3_BUCKET}api/modules.json"
+          if aws s3 cp "${PROVIDERS_URL}" /tmp/existing-registry/providers.json 2>/dev/null; then
+            echo "found=true" >> "${GITHUB_OUTPUT}"
+            aws s3 cp "${MODULES_URL}" /tmp/existing-registry/modules.json || true
+          else
+            echo "found=false" >> "${GITHUB_OUTPUT}"
+            echo "No existing registry data on S3"
+          fi
+
+      # --- Extract provider metadata ---
+      - name: "Extract registry data (breeze)"
+        env:
+          PROVIDER: ${{ inputs.provider }}
+        run: |
+          if [[ -n "${PROVIDER}" ]]; then
+            breeze registry extract-data --python 3.12 --provider "${PROVIDER}"
+          else
+            breeze registry extract-data --python 3.12
+          fi
+
+      # --- Incremental: merge new data with existing ---
+      - name: "Merge with existing registry data"
+        if: inputs.provider != '' && steps.download-existing.outputs.found == 'true'
+        run: |
+          uv run dev/registry/merge_registry_data.py \
+            --existing-providers /tmp/existing-registry/providers.json \
+            --existing-modules /tmp/existing-registry/modules.json \
+            --new-providers dev/registry/providers.json \
+            --new-modules dev/registry/modules.json \
+            --output dev/registry/
+
+      - name: "Copy breeze output to registry data"
+        run: |
+          mkdir -p registry/src/_data/versions
+          cp dev/registry/providers.json registry/src/_data/providers.json
+          cp dev/registry/modules.json registry/src/_data/modules.json
+          if [ -d dev/registry/output/versions ]; then
+            cp -r dev/registry/output/versions/* registry/src/_data/versions/
+          fi
+          # Copy provider logos extracted from providers/*/docs/integration-logos/
+          if [ -d dev/registry/logos ]; then
+            mkdir -p registry/public/logos
+            cp -r dev/registry/logos/* registry/public/logos/
+          fi
+
+      - name: "Setup pnpm"
+        uses: pnpm/action-setup@fe02b34f77f8bc703788d5817da081398fad5dd2  # v4.0.0
+        with:
+          version: 9
+
+      - name: "Setup Node.js"
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
+        with:
+          node-version: 20
+          cache: 'pnpm'
+          cache-dependency-path: 'registry/pnpm-lock.yaml'
+
+      - name: "Install Node.js dependencies"
+        working-directory: registry
+        run: pnpm install --frozen-lockfile
+
+      - name: "Build registry site"
+        working-directory: registry
+        env:
+          REGISTRY_PATH_PREFIX: "/registry/"
+        run: pnpm build
+
+      - name: "Upload registry artifact"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+        with:
+          name: registry-site
+          path: registry/_site
+          retention-days: 7
+          if-no-files-found: error
+
+      - name: "Sync registry to S3"
+        env:
+          S3_BUCKET: ${{ steps.destination.outputs.bucket }}
+        run: |
+          aws s3 sync registry/_site/ "${S3_BUCKET}" \
+            --cache-control "public, max-age=300"
+
+      - name: "Publish version metadata"
+        env:
+          S3_BUCKET: ${{ steps.destination.outputs.bucket }}
+        run: breeze registry publish-versions --s3-bucket "${S3_BUCKET}"

.github/workflows/registry-tests.yml

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Registry Tests
+on:  # yamllint disable-line rule:truthy
+  pull_request:
+    branches: [main]
+    paths:
+      - 'dev/registry/**'
+      - 'registry/**'
+      - 'providers/*/provider.yaml'
+      - 'providers/*/*/provider.yaml'
+      - '.github/workflows/registry-tests.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'dev/registry/**'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: registry-tests-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  registry-tests:
+    name: "Registry extraction tests"
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Setup Python"
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+        with:
+          python-version: "3.12"  # 3.11+ required for stdlib tomllib
+
+      - name: "Install uv"
+        run: python -m pip install uv
+
+      - name: "Install test dependencies"
+        run: uv pip install --system pytest pyyaml
+
+      - name: "Run registry extraction tests"
+        run: pytest dev/registry/tests/ -v -o "addopts="

.gitignore

@@ -284,3 +284,16 @@ _e2e_test_report.json
 
 # Allow logs folder in task-sdk-integration-tests (they logs folders are ignored above in general)
 !/task-sdk-integration-tests/logs/
+
+# Generated registry data (versions/{provider}/{version}/{metadata,parameters,connections}.json)
+# Generated by dev/registry/extract_metadata.py
+registry/src/_data/modules.json
+registry/src/_data/providers.json
+registry/src/_data/search-index.json
+# Generated by dev/registry/extract_versions.py, dev/registry/extract_parameters.py,
+# dev/registry/extract_connections.py
+registry/src/_data/versions/
+dev/registry/output/
+dev/registry/logos/
+dev/registry/modules.json
+dev/registry/providers.json