Add initial test setup running against LocalStack Snowflake

.github/workflows/changelog.yml

@@ -1,20 +1,20 @@
-name: Changelog Check
-
-on:
-  pull_request:
-    types: [opened, synchronize, labeled, unlabeled]
-    branches:
-      - main
-
-jobs:
-  check_change_log:
-    runs-on: ubuntu-latest
-    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-
-    - name: Ensure DESCRIPTION.md is updated
-      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"
+#name: Changelog Check
+#
+#on:
+#  pull_request:
+#    types: [opened, synchronize, labeled, unlabeled]
+#    branches:
+#      - main
+#
+#jobs:
+#  check_change_log:
+#    runs-on: ubuntu-latest
+#    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
+#    steps:
+#    - name: Checkout
+#      uses: actions/checkout@v3
+#      with:
+#        fetch-depth: 0
+#
+#    - name: Ensure DESCRIPTION.md is updated
+#      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"

.github/workflows/cla_bot.yml

@@ -1,29 +1,29 @@
-name: "CLA Assistant"
-on:
-  issue_comment:
-    types: [created]
-  pull_request_target:
-    types: [opened,closed,synchronize]
-
-jobs:
-  CLAssistant:
-    runs-on: ubuntu-latest
-    permissions:
-      actions: write
-      contents: write
-      pull-requests: write
-      statuses: write
-    steps:
-      - name: "CLA Assistant"
-        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action/@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
-        with:
-          path-to-signatures: 'signatures/version1.json'
-          path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
-          branch: 'main'
-          allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
-          remote-organization-name: 'snowflakedb'
-          remote-repository-name: 'cla-db'
+#name: "CLA Assistant"
+#on:
+#  issue_comment:
+#    types: [created]
+#  pull_request_target:
+#    types: [opened,closed,synchronize]
+#
+#jobs:
+#  CLAssistant:
+#    runs-on: ubuntu-latest
+#    permissions:
+#      actions: write
+#      contents: write
+#      pull-requests: write
+#      statuses: write
+#    steps:
+#      - name: "CLA Assistant"
+#        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
+#        uses: contributor-assistant/github-action/@master
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+#          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
+#        with:
+#          path-to-signatures: 'signatures/version1.json'
+#          path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
+#          branch: 'main'
+#          allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
+#          remote-organization-name: 'snowflakedb'
+#          remote-repository-name: 'cla-db'

.github/workflows/semgrep.yml

@@ -1,15 +1,15 @@
----
-name: Run semgrep checks
-
-on:
-  pull_request:
-    branches: [main]
-
-permissions:
-  contents: read
-
-jobs:
-  run-semgrep-reusable-workflow:
-    uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
-    secrets:
-      token: ${{ secrets.SEMGREP_APP_TOKEN }}
+#---
+#name: Run semgrep checks
+#
+#on:
+#  pull_request:
+#    branches: [main]
+#
+#permissions:
+#  contents: read
+#
+#jobs:
+#  run-semgrep-reusable-workflow:
+#    uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
+#    secrets:
+#      token: ${{ secrets.SEMGREP_APP_TOKEN }}

.github/workflows/snyk-issue.yml

@@ -1,33 +1,33 @@
-name: Snyk Issue
-
-on:
-  schedule:
-    - cron: '* */12 * * *'
-
-permissions:
-  contents: read
-  issues: write
-  pull-requests: write
-
-concurrency: snyk-issue
-
-jobs:
-  snyk:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout Action
-      uses: actions/checkout@v3
-      with:
-        repository: snowflakedb/whitesource-actions
-        token: ${{ secrets.whitesource_action_token }}
-        path: whitesource-actions
-    - name: Set Env
-      run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
-    - name: Jira Creation
-      uses: ./whitesource-actions/snyk-issue
-      with:
-        snyk_org: ${{ secrets.snyk_org_id_public_repo }}
-        snyk_token:  ${{ secrets.snyk_github_integration_token_public_repo }}
-        jira_token: ${{ secrets.jira_token_public_repo }}
-      env:
-        gh_token: ${{ secrets.github_token }}
+#name: Snyk Issue
+#
+#on:
+#  schedule:
+#    - cron: '* */12 * * *'
+#
+#permissions:
+#  contents: read
+#  issues: write
+#  pull-requests: write
+#
+#concurrency: snyk-issue
+#
+#jobs:
+#  snyk:
+#    runs-on: ubuntu-latest
+#    steps:
+#    - name: Checkout Action
+#      uses: actions/checkout@v3
+#      with:
+#        repository: snowflakedb/whitesource-actions
+#        token: ${{ secrets.whitesource_action_token }}
+#        path: whitesource-actions
+#    - name: Set Env
+#      run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
+#    - name: Jira Creation
+#      uses: ./whitesource-actions/snyk-issue
+#      with:
+#        snyk_org: ${{ secrets.snyk_org_id_public_repo }}
+#        snyk_token:  ${{ secrets.snyk_github_integration_token_public_repo }}
+#        jira_token: ${{ secrets.jira_token_public_repo }}
+#      env:
+#        gh_token: ${{ secrets.github_token }}

.github/workflows/snyk-pr.yml

@@ -1,37 +1,37 @@
-name: Snyk PR
-on:
-  pull_request:
-    branches:
-      - main
-
-permissions:
-  contents: read
-  issues: write
-  pull-requests: write
-
-jobs:
-  snyk:
-    runs-on: ubuntu-latest
-    if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-      with:
-        ref: ${{ github.event.pull_request.head.ref }}
-        fetch-depth: 0
-
-    - name: Checkout Action
-      uses: actions/checkout@v3
-      with:
-        repository: snowflakedb/whitesource-actions
-        token: ${{ secrets.whitesource_action_token }}
-        path: whitesource-actions
-
-    - name: Snyk Pull Request Scan Check
-      uses: ./whitesource-actions/snyk-pr
-      env:
-        pr_title: ${{ github.event.pull_request.title }}
-      with:
-        jira_token: ${{ secrets.jira_token_public_repo }}
-        gh_token: ${{ secrets.github_token }}
-        amend: false
+#name: Snyk PR
+#on:
+#  pull_request:
+#    branches:
+#      - main
+#
+#permissions:
+#  contents: read
+#  issues: write
+#  pull-requests: write
+#
+#jobs:
+#  snyk:
+#    runs-on: ubuntu-latest
+#    if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
+#    steps:
+#    - name: Checkout
+#      uses: actions/checkout@v3
+#      with:
+#        ref: ${{ github.event.pull_request.head.ref }}
+#        fetch-depth: 0
+#
+#    - name: Checkout Action
+#      uses: actions/checkout@v3
+#      with:
+#        repository: snowflakedb/whitesource-actions
+#        token: ${{ secrets.whitesource_action_token }}
+#        path: whitesource-actions
+#
+#    - name: Snyk Pull Request Scan Check
+#      uses: ./whitesource-actions/snyk-pr
+#      env:
+#        pr_title: ${{ github.event.pull_request.title }}
+#      with:
+#        jira_token: ${{ secrets.jira_token_public_repo }}
+#        gh_token: ${{ secrets.github_token }}
+#        amend: false

setup.cfg

@@ -79,6 +79,7 @@ console_scripts =
 
 [options.extras_require]
 development =
+    boto3
     Cython
     coverage
     more-itertools

test/integ/conftest.py

@@ -83,6 +83,35 @@ def print_help() -> None:
     )
 
 
+# START changes for LocalStack
+
+
+@pytest.fixture(scope="session", autouse=True)
+def create_external_volume():
+    import boto3
+
+    # create S3 bucket in LocalStack
+    s3_client = boto3.client(
+        "s3",
+        endpoint_url="http://localhost:4566",
+        aws_access_key_id="test",
+        aws_secret_access_key="test",
+    )
+    s3_client.create_bucket(Bucket="test")
+
+    # create a local external volume which is required by some of the tests
+    query = (
+        "CREATE EXTERNAL VOLUME IF NOT EXISTS python_connector_iceberg_exvol STORAGE_LOCATIONS = "
+        "((NAME = 'sl1' STORAGE_PROVIDER='S3' STORAGE_BASE_URL='s3://test' "
+        "STORAGE_AWS_ROLE_ARN='arn:aws:iam::000000000000:role/test'))"
+    )
+    connection = create_connection("default")
+    connection.cursor().execute(query)
+
+
+# END changes for LocalStack
+
+
 @pytest.fixture(scope="session")
 def is_public_test() -> bool:
     return is_public_testaccount()

test/integ/test_arrow_result.py

@@ -172,9 +172,12 @@
 RUNNING_ON_GH = os.getenv("GITHUB_ACTIONS") == "true"
 
 ICEBERG_SUPPORTED = CLOUD in ICEBERG_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
-STRUCTURED_TYPES_SUPPORTED = (
-    CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
-)
+# STRUCTURED_TYPES_SUPPORTED = (
+#     CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
+# )
+# Note whummer (2024-12-29): setting this value to False, as otherwise invalid tests are being generated,
+# using, e.g., `CREATE TABLE(c1 ARRAY(FLOAT))` which is invalid in standard case (verified against real SF)
+STRUCTURED_TYPES_SUPPORTED = False
 
 # Generate all valid test cases. By using pytest.param with an id you can
 # run a specific test case easier like so:
@@ -937,6 +940,7 @@ def test_select_date(conn_cnx):
     finish(conn_cnx, table)
 
 
+@pytest.mark.skip(reason="LocalStack - currently incompatible with Snowflake emulator")
 @pytest.mark.parametrize("scale", range(10))
 @pytest.mark.parametrize("type", ["timestampntz", "timestampltz", "timestamptz"])
 def test_select_timestamp_with_scale(conn_cnx, scale, type):

test/integ/test_cursor.py

@@ -1556,6 +1556,9 @@ def test_resultbatch(
     assert total_rows == rowcount
 
 
+@pytest.mark.skip(
+    reason="Multi-batches and large result sets currently not yet working in LocalStack"
+)
 @pytest.mark.skipolddriver(reason="new feature in v2.5.0")
 @pytest.mark.parametrize(
     "result_format,patch_path",

test/parameters.py

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved.
+#
+
+CONNECTION_PARAMETERS = {
+    "account": "test",
+    "user": "test",
+    "password": "test",
+    "database": "test",
+    "schema": "public",
+    "host": "snowflake.localhost.localstack.cloud",
+}

tox.ini

@@ -131,7 +131,7 @@ deps = flake8
 commands = flake8 {posargs}
 
 [testenv:fix_lint]
-basepython = python3.8
+# basepython = python3.8
 description = format the code base to adhere to our styles, and complain about what we cannot do automatically
 passenv =
     PROGRAMDATA