feat(FIR-46249): Firebolt Core support

.github/resources/core/README.md

@@ -0,0 +1,26 @@
+# Running tests for firebolt core over https
+
+In order to run the integration tests using https protocol, we need to bring up an nginx reverse proxy. On this proxy we will install the server certificate. In our case it will be a self signed cert (localhost.pem) and its private key (localhost-key.pem). 
+
+The cert and its key will be stored in GitHub since we cannot have secrets stored in code repository, and it would be easier to rotate them (no code changes required):
+- FIREBOLT_CORE_DEV_CERT is the certificate
+- FIREBOLT_CORE_DEV_CERT_PRIVATE_KEY is the private key
+
+This is the certificate information that we currently have. Note that it wille expire in 2 years (when it will expire we will generate a new certificate and key and set them in the Git repository secrets)
+
+Certificate Information:
+- Common Name: 
+- Subject Alternative Names: localhost
+- Organization: mkcert development certificate
+- Organization Unit: george@Mac.localdomain (George's Blue Mac)
+- Locality: 
+- State: 
+- Country: 
+- Valid From: May 22, 2025
+- Valid To: August 22, 2027
+- Issuer: mkcert george@Mac.localdomain (George's Blue Mac), mkcert development CA
+- Key Size: 2048 bit
+- Serial Number: 5b19e46ed1a5a912da2cef0129924c41
+
+
+The client will connect over https to the nginx server which will do the TLS handshake and the TLS termination. It will then call the firebolt-core over http on port 3473.

.github/resources/core/config.json

@@ -0,0 +1,7 @@
+{
+  "nodes": [
+    {
+      "host": "firebolt-core"
+    }
+  ]
+}

.github/resources/core/default.conf

@@ -0,0 +1,17 @@
+# this is the nginx configuration
+# for http we are already exposing port 3473 on the docker so it will connect to the firebolt directly without going through nginx
+
+# HTTPS server
+server {
+    listen 443 ssl;
+    server_name localhost;
+
+    # On github these the localhost
+    ssl_certificate     /etc/nginx/certs/server.pem;
+    ssl_certificate_key /etc/nginx/certs/server.key;
+
+    location / {
+        proxy_pass http://firebolt-core:3473;
+        proxy_set_header Host $host;
+    }
+}

.github/resources/core/docker-compose.yml

@@ -0,0 +1,28 @@
+name: firebolt-core
+
+services:
+  firebolt-core:
+    image: ghcr.io/firebolt-db/firebolt-core:${IMAGE_TAG}
+    container_name: firebolt-core
+    command: --node 0
+    privileged: true
+    restart: no         # equivalent to --rm (no persistence)
+    ulimits:
+      memlock: 8589934592
+    ports:
+      - 3473:3473
+    volumes:
+      # Mount the config file into the container.
+      - ${BASE_DIR}/.github/resources/core/config.json:/firebolt-core/config.json:ro
+      # Create an anonymous volume for Firebolt's internal database files. Not using a volume would have a performance impact.
+      - ${BASE_DIR}/firebolt-core:/firebolt-core/data
+
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "443:443"
+    volumes:
+      - ${BASE_DIR}/.github/resources/core/certs:/etc/nginx/certs:ro
+      - ${BASE_DIR}/.github/resources/core/default.conf:/etc/nginx/conf.d/default.conf:ro
+    depends_on:
+      - firebolt-core

.github/workflows/integration-tests-core.yml

@@ -0,0 +1,199 @@
+name: Core integration tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag_version:
+        description: 'The docker image tag for the firebolt core'
+        required: false
+        type: string
+      python_version:
+        description: 'Python version'
+        required: false
+        type: string
+        default: '3.10'
+      os_name:
+        description: 'The operating system'
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+  workflow_call:
+    inputs:
+      tag_version:
+        description: 'The docker image tag for the firebolt core'
+        required: false
+        type: string
+      python_version:
+        description: 'Python version'
+        required: false
+        type: string
+        default: '3.10'
+      os_name:
+        description: 'Operating system'
+        required: false
+        type: string
+        default: 'ubuntu-latest'
+      sendSlackNotifications:
+        description: 'Send Slack notifications on failure'
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      SLACK_BOT_TOKEN:
+        required: false
+env:
+  DEFAULT_IMAGE_TAG: ${{ vars.DEFAULT_CORE_IMAGE_TAG }}
+jobs:
+  run-core-integration-tests:
+    runs-on: ${{ inputs.os_name }}
+    env:
+      DOCKER_COMPOSE_FILE: ${{ github.workspace }}/.github/resources/core/docker-compose.yml
+      SERVICE_PORT: 3473
+      SERVICE_URL: http://localhost:3473
+      MAX_RETRIES: 30
+      RETRY_INTERVAL: 2
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+        with:
+          repository: 'firebolt-db/firebolt-python-sdk'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python_version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ".[dev]"
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Write certificate and certificate key to file
+        run: |
+          mkdir "${{ github.workspace }}/.github/resources/core/certs"
+          pip install trustme
+          # Generate a self-signed certificate for localhost
+          python3 -m trustme -d "${{ github.workspace }}/.github/resources/core/certs/"
+
+      - name: Install certs to keystore
+        run: |
+          sudo cp ${GITHUB_WORKSPACE}/.github/resources/core/certs/client.pem /usr/local/share/ca-certificates/client.crt
+          sudo update-ca-certificates
+
+      # if no image tag was passed in, then use the image tag from the defaults
+      - name: Set image tag
+        id: set-tag
+        run: |
+          IMAGE_TAG="${{ inputs.tag_version }}"
+          if [ -z "$IMAGE_TAG" ]; then
+            IMAGE_TAG="$DEFAULT_IMAGE_TAG"
+          fi
+          echo "tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Prepare docker-compose.yml
+        run: |
+          if [ ! -f "$DOCKER_COMPOSE_FILE" ]; then
+            echo "Error: Docker compose file not found at $DOCKER_COMPOSE_FILE"
+            exit 1
+          fi
+          sed -i "s|\${IMAGE_TAG}|${{ steps.set-tag.outputs.tag }}|g" "$DOCKER_COMPOSE_FILE"
+          sed -i "s|\${BASE_DIR}|${{ github.workspace }}|g" "$DOCKER_COMPOSE_FILE"
+          echo "Docker compose file prepared:"
+          cat "$DOCKER_COMPOSE_FILE"
+
+      - name: Start service container
+        run: |
+          docker compose -f "$DOCKER_COMPOSE_FILE" up -d
+          docker compose -f "$DOCKER_COMPOSE_FILE" ps
+
+      - name: Wait for service to be ready
+        run: |
+          for i in $(seq 1 $MAX_RETRIES); do
+            if curl --silent --fail "$SERVICE_URL" --data-binary "SELECT 1" | grep -q "1"; then
+              echo "Service is up and responding!"
+              exit 0
+            fi
+            echo "Waiting for service... ($i/$MAX_RETRIES)"
+            sleep $RETRY_INTERVAL
+          done
+          echo "Error: Service failed to start within timeout"
+          docker compose -f "$DOCKER_COMPOSE_FILE" logs
+          exit 1
+
+      - name: Run integration tests HTTP
+        env:
+          SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}
+          SERVICE_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}
+          DATABASE_NAME: "firebolt"
+          ENGINE_NAME: ""
+          STOPPED_ENGINE_NAME: ""
+          API_ENDPOINT: ""
+          ACCOUNT_NAME: ""
+          CORE_URL: "http://localhost:3473"
+        run: |
+          pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core"  --alluredir=allure-results/
+
+      - name: Run integration tests HTTPS
+        env:
+          SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }}
+          SERVICE_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }}
+          DATABASE_NAME: "firebolt"
+          ENGINE_NAME: ""
+          STOPPED_ENGINE_NAME: ""
+          API_ENDPOINT: ""
+          ACCOUNT_NAME: ""
+          CORE_URL: "https://localhost:443"
+        run: |
+          pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core"  --alluredir=allure-results-https/
+
+      - name: Stop container
+        if: always()
+        run: |
+          docker compose -f "$DOCKER_COMPOSE_FILE" down
+
+       # Need to pull the pages branch in order to fetch the previous runs
+      - name: Get Allure history
+        uses: actions/checkout@v4
+        if: always()
+        continue-on-error: true
+        with:
+            ref: gh-pages
+            path: gh-pages
+
+      - name: Allure Report
+        uses: firebolt-db/action-allure-report@v1
+        if: always()
+        with:
+          github-key: ${{ secrets.GITHUB_TOKEN }}
+          test-type: integration
+          allure-dir: allure-results
+          pages-branch: gh-pages
+          repository-name: python-sdk
+
+      - name: Allure Report HTTPS
+        uses: firebolt-db/action-allure-report@v1
+        if: always()
+        with:
+          github-key: ${{ secrets.GITHUB_TOKEN }}
+          test-type: integration_https
+          allure-dir: allure-results-https
+          pages-branch: gh-pages
+          repository-name: python-sdk

.github/workflows/integration-tests-v1.yml

@@ -59,7 +59,7 @@ jobs:
           API_ENDPOINT: "api.staging.firebolt.io"
           ACCOUNT_NAME: "firebolt"
         run: |
-          pytest --last-failed -n 6 --dist loadgroup --timeout_method "signal" -o log_cli=true -o log_cli_level=INFO tests/integration -k "not V2" --runslow
+          pytest --last-failed -n 6 --dist loadgroup --timeout_method "signal" -o log_cli=true -o log_cli_level=INFO tests/integration -k "not V2 and not core" --runslow
 
       - name: Save failed tests
         id: cache-tests-save

.github/workflows/integration-tests-v2.yml

@@ -57,7 +57,7 @@ jobs:
           API_ENDPOINT: "api.staging.firebolt.io"
           ACCOUNT_NAME: ${{ vars.FIREBOLT_ACCOUNT }}
         run: |
-          pytest -n 6 --dist loadgroup --timeout_method "signal" -o log_cli=true -o log_cli_level=WARNING tests/integration -k "not V1" --runslow  --alluredir=allure-results
+          pytest -n 6 --dist loadgroup --timeout_method "signal" -o log_cli=true -o log_cli_level=WARNING tests/integration -k "not V1 and not core" --runslow  --alluredir=allure-results
 
       # Need to pull the pages branch in order to fetch the previous runs
       - name: Get Allure history

.github/workflows/integration-tests.yml

@@ -23,8 +23,10 @@ jobs:
   integration-test-v2:
     uses: ./.github/workflows/integration-tests-v2.yml
     secrets: inherit
+  integration-test-core:
+    uses: ./.github/workflows/integration-tests-core.yml
   report-test-results:
-    needs: [integration-test-v1, integration-test-v2]
+    needs: [integration-test-v1, integration-test-v2, integration-test-core]
     if: always()
     runs-on: ubuntu-latest
     steps: