feat: add manifest server

.github/actions/check-docker-tag/action.yml

@@ -0,0 +1,28 @@
+name: 'Check Docker Tag Exists'
+description: 'Check if a Docker tag exists on DockerHub to prevent overwrites'
+inputs:
+  image_name:
+    description: 'Docker image name (e.g. airbyte/source-declarative-manifest)'
+    required: true
+  tag:
+    description: 'Docker tag to check'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: "Check for existing tag (${{ inputs.image_name }}:${{ inputs.tag }})"
+      shell: bash
+      run: |
+        image="${{ inputs.image_name }}"
+        tag_input="${{ inputs.tag }}"
+        if [ -z "$image" ] || [ -z "$tag_input" ]; then
+          echo "Error: image_name and tag are required."
+          exit 1
+        fi
+        tag="${image}:${tag_input}"
+        echo "Checking if tag '$tag' exists on DockerHub..."
+        if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
+          echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
+          exit 1
+        fi
+        echo "No existing tag '$tag' found. Proceeding with publish."

.github/workflows/docker-build-check.yml

@@ -1,13 +1,15 @@
 name: Docker Build Check
+permissions:
+  contents: read
 
 on:
   pull_request:
     branches:
       - main
 
 jobs:
-  docker-build-check:
-    name: SDM Docker Image Build # Renamed job to be more descriptive
+  sdm-docker-build-check:
+    name: SDM Docker Image Build
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
@@ -42,3 +44,29 @@ jobs:
           push: false
           tags: airbyte/source-declarative-manifest:pr-${{ github.event.pull_request.number }}
           outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=SDM Docker image for PR ${{ github.event.pull_request.number }}
+
+  manifest-server-docker-build-check:
+    name: Manifest Server Docker Image Build
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Manifest Server Docker image for multiple platforms
+        id: manifest-server-build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: airbyte_cdk/manifest_server/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: false
+          tags: airbyte/manifest-server:pr-${{ github.event.pull_request.number }}
+          outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Manifest Server Docker image for PR ${{ github.event.pull_request.number }}

.github/workflows/pypi_publish.yml → .github/workflows/publish.yml

@@ -6,6 +6,8 @@
 # we have to also update the Trusted Publisher settings on PyPI.
 
 name: CDK Publish
+permissions:
+  contents: read
 
 on:
   push:
@@ -31,6 +33,11 @@ on:
         type: boolean
         required: true
         default: true
+      publish_manifest_server:
+        description: "Publish Manifest Server to DockerHub. If true, the workflow will publish the Manifest Server to DockerHub."
+        type: boolean
+        required: true
+        default: true
       update_connector_builder:
         description: "Update Connector Builder. If true, the workflow will create a PR to bump the CDK version used by Connector Builder."
         type: boolean
@@ -204,18 +211,10 @@ jobs:
 
       - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
         if: env.VERSION != ''
-        run: |
-          tag="airbyte/source-declarative-manifest:${{ env.VERSION }}"
-          if [ -z "$tag" ]; then
-            echo "Error: VERSION is not set. Ensure the tag follows the format 'refs/tags/vX.Y.Z'."
-            exit 1
-          fi
-          echo "Checking if tag '$tag' exists on DockerHub..."
-          if DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect "$tag" > /dev/null 2>&1; then
-            echo "The tag '$tag' already exists on DockerHub. Skipping publish to prevent overwrite."
-            exit 1
-          fi
-          echo "No existing tag '$tag' found. Proceeding with publish."
+        uses: ./.github/actions/check-docker-tag
+        with:
+          image_name: airbyte/source-declarative-manifest
+          tag: ${{ env.VERSION }}
 
       - name: "Build and push (sha tag: '${{ github.sha }}')"
         # Only run if the version is not set
@@ -250,6 +249,90 @@ jobs:
           tags: |
             airbyte/source-declarative-manifest:latest
 
+  publish_manifest_server:
+    name: Publish Manifest Server to DockerHub
+    if: >
+      (github.event_name == 'push' &&
+       startsWith(github.ref, 'refs/tags/v')) ||
+      (github.event_name == 'workflow_dispatch' &&
+       github.event.inputs.publish_manifest_server == 'true'
+      )
+    runs-on: ubuntu-24.04
+    needs: [build]
+    environment:
+      name: DockerHub
+      url: https://hub.docker.com/r/airbyte/manifest-server/tags
+    env:
+      VERSION: ${{ needs.build.outputs.VERSION }}
+      IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      # We need to download the build artifact again because the previous job was on a different runner
+      - name: Download Build Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: Packages-${{ github.run_id }}
+          path: dist
+
+      - name: Set up QEMU for multi-platform builds
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: "Check for existing tag (version: ${{ env.VERSION || 'none' }} )"
+        if: env.VERSION != ''
+        uses: ./.github/actions/check-docker-tag
+        with:
+          image_name: airbyte/manifest-server
+          tag: ${{ env.VERSION }}
+
+      - name: "Build and push (sha tag: '${{ github.sha }}')"
+        # Only run if the version is not set
+        if: env.VERSION == ''
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: airbyte_cdk/manifest_server/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/manifest-server:${{ github.sha }}
+
+      - name: "Build and push (version tag: ${{ env.VERSION || 'none'}})"
+        # Only run if the version is set
+        if: env.VERSION != ''
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: airbyte_cdk/manifest_server/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/manifest-server:${{ env.VERSION }}
+
+      - name: Build and push ('latest' tag)
+        # Only run if version is set and IS_PRERELEASE is false
+        if: env.VERSION != '' && env.IS_PRERELEASE == 'false'
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: airbyte_cdk/manifest_server/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: |
+            airbyte/manifest-server:latest
+
   update-connector-builder:
     # Create a PR against the Builder, to update the CDK version that it uses.
     # In the future, Builder may use the SDM docker image instead of the Python CDK package.

airbyte_cdk/manifest_server/Dockerfile

@@ -0,0 +1,45 @@
+# Dockerfile for the Airbyte Manifest Server.
+#
+# This Dockerfile should be built from the root of the repository.
+#
+# Example:
+# docker build -f airbyte_cdk/manifest_server/Dockerfile -t airbyte/manifest-server .
+
+FROM python:3.12-slim-bookworm
+
+# Install git (needed for dynamic versioning) and poetry
+RUN apt-get update && \
+    apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/* && \
+    pip install poetry==1.8.3
+
+# Configure poetry to not create virtual environments and disable interactive mode
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+# Copy poetry files (build from project root)
+COPY pyproject.toml poetry.lock ./
+
+# Copy the project source code (needed for dynamic versioning)
+COPY . /app
+
+# Install dependencies and package directly to system Python
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR \
+    poetry config virtualenvs.create false && \
+    poetry install --extras manifest-server
+
+# Create a non-root user and group
+RUN groupadd --gid 1000 airbyte && \
+    useradd --uid 1000 --gid airbyte --shell /bin/bash --create-home airbyte
+
+# Change ownership
+RUN chown -R airbyte:airbyte /app
+
+# Run app as non-root user
+USER airbyte:airbyte
+
+EXPOSE 8080
+
+CMD ["uvicorn", "airbyte_cdk.manifest_server.app:app", "--host", "0.0.0.0", "--port", "8080"]