diff --git a/.github/resources/core/README.md b/.github/resources/core/README.md deleted file mode 100644 index c5e5337933..0000000000 --- a/.github/resources/core/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Running tests for firebolt core over https - -In order to run the integration tests using https protocol, we need to bring up an nginx reverse proxy. On this proxy we will install the server certificate. In our case it will be a self signed cert (localhost.pem) and its private key (localhost-key.pem). - -The cert and its key will be stored in GitHub since we cannot have secrets stored in code repository, and it would be easier to rotate them (no code changes required): -- FIREBOLT_CORE_DEV_CERT is the certificate -- FIREBOLT_CORE_DEV_CERT_PRIVATE_KEY is the private key - -This is the certificate information that we currently have. Note that it wille expire in 2 years (when it will expire we will generate a new certificate and key and set them in the Git repository secrets) - -Certificate Information: -- Common Name: -- Subject Alternative Names: localhost -- Organization: mkcert development certificate -- Organization Unit: george@Mac.localdomain (George's Blue Mac) -- Locality: -- State: -- Country: -- Valid From: May 22, 2025 -- Valid To: August 22, 2027 -- Issuer: mkcert george@Mac.localdomain (George's Blue Mac), mkcert development CA -- Key Size: 2048 bit -- Serial Number: 5b19e46ed1a5a912da2cef0129924c41 - - -The client will connect over https to the nginx server which will do the TLS handshake and the TLS termination. It will then call the firebolt-core over http on port 3473. diff --git a/.github/resources/core/config.json b/.github/resources/core/config.json deleted file mode 100644 index d7fe906fd1..0000000000 --- a/.github/resources/core/config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "nodes": [ - { - "host": "firebolt-core" - } - ] -} \ No newline at end of file diff --git a/.github/resources/core/default.conf b/.github/resources/core/default.conf deleted file mode 100644 index 81591b3873..0000000000 --- a/.github/resources/core/default.conf +++ /dev/null @@ -1,17 +0,0 @@ -# this is the nginx configuration -# for http we are already exposing port 3473 on the docker so it will connect to the firebolt directly without going through nginx - -# HTTPS server -server { - listen 443 ssl; - server_name localhost; - - # On github these the localhost - ssl_certificate /etc/nginx/certs/server.pem; - ssl_certificate_key /etc/nginx/certs/server.key; - - location / { - proxy_pass http://firebolt-core:3473; - proxy_set_header Host $host; - } -} \ No newline at end of file diff --git a/.github/resources/core/docker-compose.yml b/.github/resources/core/docker-compose.yml deleted file mode 100644 index 85092e6add..0000000000 --- a/.github/resources/core/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: firebolt-core - -services: - firebolt-core: - image: ghcr.io/firebolt-db/firebolt-core:${IMAGE_TAG} - container_name: firebolt-core - command: --node 0 - privileged: true - restart: no # equivalent to --rm (no persistence) - ulimits: - memlock: 8589934592 - ports: - - 3473:3473 - volumes: - # Mount the config file into the container. - - ${BASE_DIR}/.github/resources/core/config.json:/firebolt-core/config.json:ro - # Create an anonymous volume for Firebolt's internal database files. Not using a volume would have a performance impact. - - ${BASE_DIR}/firebolt-core:/firebolt-core/data - - nginx: - image: nginx:alpine - ports: - - "443:443" - volumes: - - ${BASE_DIR}/.github/resources/core/certs:/etc/nginx/certs:ro - - ${BASE_DIR}/.github/resources/core/default.conf:/etc/nginx/conf.d/default.conf:ro - depends_on: - - firebolt-core \ No newline at end of file diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index cf9deca010..fa26a7faa3 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -41,23 +41,21 @@ on: secrets: SLACK_BOT_TOKEN: required: false -env: - DEFAULT_IMAGE_TAG: ${{ vars.DEFAULT_CORE_IMAGE_TAG }} jobs: run-core-integration-tests: runs-on: ${{ inputs.os_name }} - env: - DOCKER_COMPOSE_FILE: ${{ github.workspace }}/.github/resources/core/docker-compose.yml - SERVICE_PORT: 3473 - SERVICE_URL: http://localhost:3473 - MAX_RETRIES: 30 - RETRY_INTERVAL: 2 steps: - name: Check out code uses: actions/checkout@v4 with: repository: 'firebolt-db/firebolt-python-sdk' + - name: Setup Firebolt Core + id: setup-core + uses: firebolt-db/action-setup-core@main + with: + tag_version: ${{ inputs.tag_version || vars.DEFAULT_CORE_IMAGE_TAG }} + - name: Set up Python uses: actions/setup-python@v5 with: @@ -68,76 +66,6 @@ jobs: python -m pip install --upgrade pip pip install ".[dev]" - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Write certificate and certificate key to file - run: | - mkdir "${{ github.workspace }}/.github/resources/core/certs" - pip install trustme - # Generate a self-signed certificate for localhost - python3 -m trustme -d "${{ github.workspace }}/.github/resources/core/certs/" - - - name: Install certs to keystore - run: | - sudo cp ${GITHUB_WORKSPACE}/.github/resources/core/certs/client.pem /usr/local/share/ca-certificates/client.crt - sudo update-ca-certificates - - # if no image tag was passed in, then use the image tag from the defaults - - name: Set image tag - id: set-tag - run: | - IMAGE_TAG="${{ inputs.tag_version }}" - if [ -z "$IMAGE_TAG" ]; then - IMAGE_TAG="$DEFAULT_IMAGE_TAG" - fi - echo "tag=$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Prepare docker-compose.yml - run: | - if [ ! -f "$DOCKER_COMPOSE_FILE" ]; then - echo "Error: Docker compose file not found at $DOCKER_COMPOSE_FILE" - exit 1 - fi - sed -i "s|\${IMAGE_TAG}|${{ steps.set-tag.outputs.tag }}|g" "$DOCKER_COMPOSE_FILE" - sed -i "s|\${BASE_DIR}|${{ github.workspace }}|g" "$DOCKER_COMPOSE_FILE" - echo "Docker compose file prepared:" - cat "$DOCKER_COMPOSE_FILE" - - - name: Start service container - run: | - docker compose -f "$DOCKER_COMPOSE_FILE" up -d - docker compose -f "$DOCKER_COMPOSE_FILE" ps - - - name: Wait for service to be ready - run: | - for i in $(seq 1 $MAX_RETRIES); do - if curl --silent --fail "$SERVICE_URL" --data-binary "SELECT 1" | grep -q "1"; then - echo "Service is up and responding!" - exit 0 - fi - echo "Waiting for service... ($i/$MAX_RETRIES)" - sleep $RETRY_INTERVAL - done - echo "Error: Service failed to start within timeout" - docker compose -f "$DOCKER_COMPOSE_FILE" logs - exit 1 - - name: Run integration tests HTTP env: SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} @@ -147,7 +75,7 @@ jobs: STOPPED_ENGINE_NAME: "" API_ENDPOINT: "" ACCOUNT_NAME: "" - CORE_URL: "http://localhost:3473" + CORE_URL: ${{ steps.setup-core.outputs.service_url }} run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results/ @@ -160,15 +88,10 @@ jobs: STOPPED_ENGINE_NAME: "" API_ENDPOINT: "" ACCOUNT_NAME: "" - CORE_URL: "https://localhost:443" + CORE_URL: ${{ steps.setup-core.outputs.service_https_url }} run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results-https/ - - name: Stop container - if: always() - run: | - docker compose -f "$DOCKER_COMPOSE_FILE" down - # Need to pull the pages branch in order to fetch the previous runs - name: Get Allure history uses: actions/checkout@v4 @@ -184,7 +107,7 @@ jobs: continue-on-error: true with: github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration + test-type: core allure-dir: allure-results pages-branch: gh-pages repository-name: python-sdk @@ -195,7 +118,7 @@ jobs: continue-on-error: true with: github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration_https + test-type: core_https allure-dir: allure-results-https pages-branch: gh-pages repository-name: python-sdk \ No newline at end of file