diff --git a/.github/resources/core/default.conf b/.github/resources/core/default.conf index 7bdeac8950..81591b3873 100644 --- a/.github/resources/core/default.conf +++ b/.github/resources/core/default.conf @@ -7,8 +7,8 @@ server { server_name localhost; # On github these the localhost - ssl_certificate /etc/nginx/certs/localhost.pem; - ssl_certificate_key /etc/nginx/certs/localhost-key.pem; + ssl_certificate /etc/nginx/certs/server.pem; + ssl_certificate_key /etc/nginx/certs/server.key; location / { proxy_pass http://firebolt-core:3473; diff --git a/.github/workflows/integration-tests-core.yml b/.github/workflows/integration-tests-core.yml index f8aaf67fdf..894ef659d1 100644 --- a/.github/workflows/integration-tests-core.yml +++ b/.github/workflows/integration-tests-core.yml @@ -11,7 +11,7 @@ on: description: 'Python version' required: false type: string - default: '3.9' + default: '3.10' os_name: description: 'The operating system' required: false @@ -27,7 +27,7 @@ on: description: 'Python version' required: false type: string - default: '3.9' + default: '3.10' os_name: description: 'Operating system' required: false @@ -89,17 +89,14 @@ jobs: - name: Write certificate and certificate key to file run: | mkdir "${{ github.workspace }}/.github/resources/core/certs" - echo "${{ secrets.FIREBOLT_CORE_DEV_CERT_PRIVATE_KEY }}" > ${{ github.workspace }}/.github/resources/core/certs/localhost-key.pem - echo "${{ secrets.FIREBOLT_CORE_DEV_CERT }}" > ${{ github.workspace }}/.github/resources/core/certs/localhost.pem + pip install trustme + # Generate a self-signed certificate for localhost + python3 -m trustme -d "${{ github.workspace }}/.github/resources/core/certs/" - name: Install certs to keystore run: | - sudo keytool -importcert \ - -keystore "$JAVA_HOME/lib/security/cacerts" \ - -storepass changeit \ - -file "${{ github.workspace }}/.github/resources/core/certs/localhost.pem" \ - -alias dev-localhost \ - -noprompt + sudo cp ${GITHUB_WORKSPACE}/.github/resources/core/certs/client.pem /usr/local/share/ca-certificates/client.crt + sudo update-ca-certificates # if no image tag was passed in, then use the image tag from the defaults - name: Set image tag @@ -154,6 +151,19 @@ jobs: run: | pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results/ + - name: Run integration tests HTTPS + env: + SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} + SERVICE_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }} + DATABASE_NAME: "firebolt" + ENGINE_NAME: "" + STOPPED_ENGINE_NAME: "" + API_ENDPOINT: "" + ACCOUNT_NAME: "" + CORE_URL: "https://localhost:443" + run: | + pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results-https/ + - name: Stop container if: always() run: | @@ -176,4 +186,14 @@ jobs: test-type: integration allure-dir: allure-results pages-branch: gh-pages + repository-name: python-sdk + + - name: Allure Report HTTPS + uses: firebolt-db/action-allure-report@v1 + if: always() + with: + github-key: ${{ secrets.GITHUB_TOKEN }} + test-type: integration_https + allure-dir: allure-results-https + pages-branch: gh-pages repository-name: python-sdk \ No newline at end of file diff --git a/.github/workflows/integration-tests-v1.yml b/.github/workflows/integration-tests-v1.yml index 6aafb1a4bb..908f52385b 100644 --- a/.github/workflows/integration-tests-v1.yml +++ b/.github/workflows/integration-tests-v1.yml @@ -1,179 +1,71 @@ -name: Core integration tests - +name: Integration tests FB 1.0 on: workflow_dispatch: - inputs: - tag_version: - description: 'The docker image tag for the firebolt core' - required: false - type: string - python_version: - description: 'Python version' - required: false - type: string - default: '3.10' - os_name: - description: 'The operating system' - required: false - type: string - default: 'ubuntu-latest' workflow_call: - inputs: - tag_version: - description: 'The docker image tag for the firebolt core' - required: false - type: string - python_version: - description: 'Python version' - required: false - type: string - default: '3.10' - os_name: - description: 'Operating system' - required: false - type: string - default: 'ubuntu-latest' - sendSlackNotifications: - description: 'Send Slack notifications on failure' - required: false - type: boolean - default: false secrets: - SLACK_BOT_TOKEN: - required: false -env: - DEFAULT_IMAGE_TAG: ${{ vars.DEFAULT_CORE_IMAGE_TAG }} + FIREBOLT_STG_USERNAME: + required: true + FIREBOLT_STG_PASSWORD: + required: true + SERVICE_ID_STG: + required: true + SERVICE_SECRET_STG: + required: true jobs: - run-core-integration-tests: - runs-on: ${{ inputs.os_name }} - env: - DOCKER_COMPOSE_FILE: ${{ github.workspace }}/.github/resources/core/docker-compose.yml - SERVICE_PORT: 3473 - SERVICE_URL: http://localhost:3473 - MAX_RETRIES: 30 - RETRY_INTERVAL: 2 + tests: + runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 - with: - repository: 'firebolt-db/firebolt-python-sdk' + uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 with: - python-version: ${{ inputs.python_version }} + python-version: 3.8 - name: Install dependencies run: | python -m pip install --upgrade pip pip install ".[dev]" - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + - name: Setup database and engine + id: setup + uses: firebolt-db/integration-testing-setup@v1 with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Cache Docker layers - uses: actions/cache@v3 + firebolt-username: ${{ secrets.FIREBOLT_STG_USERNAME }} + firebolt-password: ${{ secrets.FIREBOLT_STG_PASSWORD }} + api-endpoint: "api.staging.firebolt.io" + region: "us-east-1" + + - name: Restore cached failed tests + id: cache-tests-restore + uses: actions/cache/restore@v3 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - name: Write certificate and certificate key to file - run: | - mkdir "${{ github.workspace }}/.github/resources/core/certs" - echo "${{ secrets.FIREBOLT_CORE_DEV_CERT_PRIVATE_KEY }}" > ${{ github.workspace }}/.github/resources/core/certs/localhost-key.pem - echo "${{ secrets.FIREBOLT_CORE_DEV_CERT }}" > ${{ github.workspace }}/.github/resources/core/certs/localhost.pem - - - name: Install certs to keystore - run: | - sudo keytool -importcert \ - -keystore "$JAVA_HOME/lib/security/cacerts" \ - -storepass changeit \ - -file "${{ github.workspace }}/.github/resources/core/certs/localhost.pem" \ - -alias dev-localhost \ - -noprompt - - # if no image tag was passed in, then use the image tag from the defaults - - name: Set image tag - id: set-tag - run: | - IMAGE_TAG="${{ inputs.tag_version }}" - if [ -z "$IMAGE_TAG" ]; then - IMAGE_TAG="$DEFAULT_IMAGE_TAG" - fi - echo "tag=$IMAGE_TAG" >> $GITHUB_OUTPUT + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ runner.os }}-pytest-restore-failed-${{ github.ref }}-${{ github.sha }} - - name: Prepare docker-compose.yml - run: | - if [ ! -f "$DOCKER_COMPOSE_FILE" ]; then - echo "Error: Docker compose file not found at $DOCKER_COMPOSE_FILE" - exit 1 - fi - sed -i "s|\${IMAGE_TAG}|${{ steps.set-tag.outputs.tag }}|g" "$DOCKER_COMPOSE_FILE" - sed -i "s|\${BASE_DIR}|${{ github.workspace }}|g" "$DOCKER_COMPOSE_FILE" - echo "Docker compose file prepared:" - cat "$DOCKER_COMPOSE_FILE" - - - name: Start service container - run: | - docker compose -f "$DOCKER_COMPOSE_FILE" up -d - docker compose -f "$DOCKER_COMPOSE_FILE" ps - - - name: Wait for service to be ready - run: | - for i in $(seq 1 $MAX_RETRIES); do - if curl --silent --fail "$SERVICE_URL" --data-binary "SELECT 1" | grep -q "1"; then - echo "Service is up and responding!" - exit 0 - fi - echo "Waiting for service... ($i/$MAX_RETRIES)" - sleep $RETRY_INTERVAL - done - echo "Error: Service failed to start within timeout" - docker compose -f "$DOCKER_COMPOSE_FILE" logs - exit 1 - - - name: Run integration tests HTTP + - name: Run integration tests env: - SERVICE_ID: ${{ secrets.FIREBOLT_CLIENT_ID_STG_NEW_IDN }} - SERVICE_SECRET: ${{ secrets.FIREBOLT_CLIENT_SECRET_STG_NEW_IDN }} - DATABASE_NAME: "firebolt" - ENGINE_NAME: "" - STOPPED_ENGINE_NAME: "" - API_ENDPOINT: "" - ACCOUNT_NAME: "" - CORE_URL: "http://localhost:3473" - run: | - pytest -o log_cli=true -o log_cli_level=WARNING tests/integration -k "core" --alluredir=allure-results/ - - - name: Stop container - if: always() + USER_NAME: ${{ secrets.FIREBOLT_STG_USERNAME}} + PASSWORD: ${{ secrets.FIREBOLT_STG_PASSWORD }} + SERVICE_ID: ${{ secrets.SERVICE_ID_STG }} + SERVICE_SECRET: ${{ secrets.SERVICE_SECRET_STG }} + DATABASE_NAME: ${{ steps.setup.outputs.database_name }} + ENGINE_NAME: ${{ steps.setup.outputs.engine_name }} + ENGINE_URL: ${{ steps.setup.outputs.engine_url }} + STOPPED_ENGINE_NAME: ${{ steps.setup.outputs.stopped_engine_name }} + STOPPED_ENGINE_URL: ${{ steps.setup.outputs.stopped_engine_url }} + API_ENDPOINT: "api.staging.firebolt.io" + ACCOUNT_NAME: "firebolt" run: | - docker compose -f "$DOCKER_COMPOSE_FILE" down - - # Need to pull the pages branch in order to fetch the previous runs - - name: Get Allure history - uses: actions/checkout@v4 - if: always() - continue-on-error: true - with: - ref: gh-pages - path: gh-pages + pytest --last-failed -n 6 --dist loadgroup --timeout_method "signal" -o log_cli=true -o log_cli_level=INFO tests/integration -k "not V2" --runslow - - name: Allure Report - uses: firebolt-db/action-allure-report@v1 - if: always() + - name: Save failed tests + id: cache-tests-save + uses: actions/cache/save@v3 + if: failure() with: - github-key: ${{ secrets.GITHUB_TOKEN }} - test-type: integration - allure-dir: allure-results - pages-branch: gh-pages - repository-name: python-sdk \ No newline at end of file + path: | + .pytest_cache/v/cache/lastfailed + key: ${{ steps.cache-tests-restore.outputs.cache-primary-key }} \ No newline at end of file diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index cb0a63dbab..839d112dee 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -23,8 +23,10 @@ jobs: integration-test-v2: uses: ./.github/workflows/integration-tests-v2.yml secrets: inherit + integration-test-core: + uses: ./.github/workflows/integration-tests-core.yml report-test-results: - needs: [integration-test-v1, integration-test-v2] + needs: [integration-test-v1, integration-test-v2, integration-test-core] if: always() runs-on: ubuntu-latest steps: diff --git a/docsrc/Connecting_and_queries.rst b/docsrc/Connecting_and_queries.rst index 107495700c..5fb815ed1e 100644 --- a/docsrc/Connecting_and_queries.rst +++ b/docsrc/Connecting_and_queries.rst @@ -207,6 +207,30 @@ To get started, follow the steps below: `official docs `_. +**2.1. Connecting to an HTTPS Server** + + If you are connecting to an HTTPS server running Firebolt Core, ensure the server's certificate is properly configured. Follow these steps: + + - **Obtain the Certificate**: Ensure you have the certificate for the HTTPS server you are connecting to. + + - **Install the Certificate in the System Certificate Store**: For Ubuntu users, copy the certificate to `/usr/local/share/ca-certificates/` and run the following command: + + ```bash + sudo update-ca-certificates + ``` + + .. note:: + Installing the certificate is also possible on other systems, but you need to use the correct path for your operating system's certificate store. + + - **Provide the Certificate via Environment Variable**: Alternatively, set the `SSL_CERT_FILE` environment variable to the path of your certificate file. For example: + + ```bash + export SSL_CERT_FILE=/path/to/your/certificate.pem + ``` + + - **Python Version Considerations**: The system certificate store is only available for users running Python 3.10 and above. If you are using an older version of Python, you must explicitly set the `SSL_CERT_FILE` environment variable to use the certificate. + + **3. Execute commands using the cursor** The ``cursor`` object can be used to send queries and commands to your Firebolt @@ -832,4 +856,3 @@ function will raise a ``QueryTimeoutError`` exception. ) **Warning**: If running multiple queries, and one of queries times out, all the previous queries will not be rolled back and their result will persist. All the remaining queries will be cancelled. - diff --git a/setup.cfg b/setup.cfg index 04140ea41f..d2c5a97585 100755 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ install_requires = readerwriterlock>=1.0.9 sqlparse>=0.4.2 trio>=0.22.0 + truststore>=0.10;python_version>="3.10" python_requires = >=3.8 include_package_data = True package_dir = diff --git a/src/firebolt/async_db/connection.py b/src/firebolt/async_db/connection.py index 0d40d4faa8..202aa7530e 100644 --- a/src/firebolt/async_db/connection.py +++ b/src/firebolt/async_db/connection.py @@ -1,7 +1,8 @@ from __future__ import annotations +from ssl import SSLContext from types import TracebackType -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type, Union from httpx import Timeout @@ -28,6 +29,7 @@ FireboltError, ) from firebolt.utils.firebolt_core import ( + get_core_certificate_context, parse_firebolt_core_url, validate_firebolt_core_parameters, ) @@ -419,6 +421,10 @@ def connect_core( """ connection_params = parse_firebolt_core_url(connection_url) + ctx: Union[SSLContext, bool] = True # Default context + if connection_params.scheme == "https": + ctx = get_core_certificate_context() + verified_url = connection_params.geturl() client = AsyncClientV2( auth=auth, @@ -426,6 +432,7 @@ def connect_core( base_url=verified_url, timeout=Timeout(DEFAULT_TIMEOUT_SECONDS, read=None), headers={"User-Agent": user_agent_header}, + verify=ctx, ) return Connection( diff --git a/src/firebolt/client/client.py b/src/firebolt/client/client.py index 3515255241..b8d7128024 100644 --- a/src/firebolt/client/client.py +++ b/src/firebolt/client/client.py @@ -109,7 +109,7 @@ def __init__(self, *args: Any, **kwargs: Any): super().__init__( *args, **kwargs, - transport=KeepaliveTransport(), + transport=KeepaliveTransport(verify=kwargs.get("verify", True)), ) @property @@ -276,7 +276,7 @@ def __init__(self, *args: Any, **kwargs: Any): super().__init__( *args, **kwargs, - transport=AsyncKeepaliveTransport(), + transport=AsyncKeepaliveTransport(verify=kwargs.get("verify", True)), ) @property diff --git a/src/firebolt/db/connection.py b/src/firebolt/db/connection.py index eded811602..ef7b5ff62d 100644 --- a/src/firebolt/db/connection.py +++ b/src/firebolt/db/connection.py @@ -1,8 +1,9 @@ from __future__ import annotations import logging +from ssl import SSLContext from types import TracebackType -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type, Union from warnings import warn from httpx import Timeout @@ -29,6 +30,7 @@ FireboltError, ) from firebolt.utils.firebolt_core import ( + get_core_certificate_context, parse_firebolt_core_url, validate_firebolt_core_parameters, ) @@ -425,6 +427,10 @@ def connect_core( """ connection_params = parse_firebolt_core_url(connection_url) + ctx: Union[SSLContext, bool] = True # Default context + if connection_params.scheme == "https": + ctx = get_core_certificate_context() + verified_url = connection_params.geturl() client = ClientV2( @@ -433,6 +439,7 @@ def connect_core( base_url=verified_url, timeout=Timeout(DEFAULT_TIMEOUT_SECONDS, read=None), headers={"User-Agent": user_agent_header}, + verify=ctx, ) return Connection( diff --git a/src/firebolt/utils/firebolt_core.py b/src/firebolt/utils/firebolt_core.py index 660154f33d..8af4f97fa6 100644 --- a/src/firebolt/utils/firebolt_core.py +++ b/src/firebolt/utils/firebolt_core.py @@ -1,9 +1,31 @@ -from typing import Optional +import os +import sys +from ssl import PROTOCOL_TLS_CLIENT, SSLContext, create_default_context +from typing import Optional, Union from urllib.parse import ParseResult, urlparse from firebolt.utils.exception import ConfigurationError +def get_core_certificate_context() -> Union[SSLContext, bool]: + """Get the SSL context for Firebolt Core connections.""" + ctx: Union[SSLContext, bool] = True # Default context for SSL verification + if os.getenv("SSL_CERT_FILE"): + ctx = create_default_context(cafile=os.getenv("SSL_CERT_FILE")) + elif sys.version_info >= (3, 10): + # Can import truststore only if python is 3.10 or higher + import truststore + + ctx = truststore.SSLContext(PROTOCOL_TLS_CLIENT) + else: + raise ConfigurationError( + "Not able to use system certificate store for Firebolt Core." + " on Python < 3.10, you may need to set" + " SSL_CERT_FILE environment variable pointing to your .pem file." + ) + return ctx + + def validate_firebolt_core_parameters( account_name: Optional[str] = None, engine_name: Optional[str] = None,