Skip to content

Commit d623456

Browse files
whummerwhummer
committed
add initial test setup running against LocalStack Snowflake (#1)
1 parent db2a10f commit d623456

File tree

12 files changed

+443
-535
lines changed

12 files changed

+443
-535
lines changed

‎.github/workflows/build_test.yml‎

Lines changed: 255 additions & 400 deletions
Large diffs are not rendered by default.

‎.github/workflows/changelog.yml‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
1-
name: Changelog Check
2-
3-
on:
4-
pull_request:
5-
types: [opened, synchronize, labeled, unlabeled]
6-
branches:
7-
- main
8-
9-
jobs:
10-
check_change_log:
11-
runs-on: ubuntu-latest
12-
if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
13-
steps:
14-
- name: Checkout
15-
uses: actions/checkout@v3
16-
with:
17-
fetch-depth: 0
18-
19-
- name: Ensure DESCRIPTION.md is updated
20-
run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"
1+
#name: Changelog Check
2+
#
3+
#on:
4+
# pull_request:
5+
# types: [opened, synchronize, labeled, unlabeled]
6+
# branches:
7+
# - main
8+
#
9+
#jobs:
10+
# check_change_log:
11+
# runs-on: ubuntu-latest
12+
# if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
13+
# steps:
14+
# - name: Checkout
15+
# uses: actions/checkout@v3
16+
# with:
17+
# fetch-depth: 0
18+
#
19+
# - name: Ensure DESCRIPTION.md is updated
20+
# run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md"

‎.github/workflows/cla_bot.yml‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,29 @@
1-
name: "CLA Assistant"
2-
on:
3-
issue_comment:
4-
types: [created]
5-
pull_request_target:
6-
types: [opened,closed,synchronize]
7-
8-
jobs:
9-
CLAssistant:
10-
runs-on: ubuntu-latest
11-
permissions:
12-
actions: write
13-
contents: write
14-
pull-requests: write
15-
statuses: write
16-
steps:
17-
- name: "CLA Assistant"
18-
if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
19-
uses: contributor-assistant/github-action/@master
20-
env:
21-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22-
PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
23-
with:
24-
path-to-signatures: 'signatures/version1.json'
25-
path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
26-
branch: 'main'
27-
allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
28-
remote-organization-name: 'snowflakedb'
29-
remote-repository-name: 'cla-db'
1+
#name: "CLA Assistant"
2+
#on:
3+
# issue_comment:
4+
# types: [created]
5+
# pull_request_target:
6+
# types: [opened,closed,synchronize]
7+
#
8+
#jobs:
9+
# CLAssistant:
10+
# runs-on: ubuntu-latest
11+
# permissions:
12+
# actions: write
13+
# contents: write
14+
# pull-requests: write
15+
# statuses: write
16+
# steps:
17+
# - name: "CLA Assistant"
18+
# if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
19+
# uses: contributor-assistant/github-action/@master
20+
# env:
21+
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
22+
# PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
23+
# with:
24+
# path-to-signatures: 'signatures/version1.json'
25+
# path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
26+
# branch: 'main'
27+
# allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa'
28+
# remote-organization-name: 'snowflakedb'
29+
# remote-repository-name: 'cla-db'

‎.github/workflows/semgrep.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
1-
---
2-
name: Run semgrep checks
3-
4-
on:
5-
pull_request:
6-
branches: [main]
7-
8-
permissions:
9-
contents: read
10-
11-
jobs:
12-
run-semgrep-reusable-workflow:
13-
uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
14-
secrets:
15-
token: ${{ secrets.SEMGREP_APP_TOKEN }}
1+
#---
2+
#name: Run semgrep checks
3+
#
4+
#on:
5+
# pull_request:
6+
# branches: [main]
7+
#
8+
#permissions:
9+
# contents: read
10+
#
11+
#jobs:
12+
# run-semgrep-reusable-workflow:
13+
# uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main
14+
# secrets:
15+
# token: ${{ secrets.SEMGREP_APP_TOKEN }}

‎.github/workflows/snyk-issue.yml‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,33 @@
1-
name: Snyk Issue
2-
3-
on:
4-
schedule:
5-
- cron: '* */12 * * *'
6-
7-
permissions:
8-
contents: read
9-
issues: write
10-
pull-requests: write
11-
12-
concurrency: snyk-issue
13-
14-
jobs:
15-
snyk:
16-
runs-on: ubuntu-latest
17-
steps:
18-
- name: checkout action
19-
uses: actions/checkout@v4
20-
with:
21-
repository: snowflakedb/whitesource-actions
22-
token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
23-
path: whitesource-actions
24-
- name: set-env
25-
run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
26-
- name: Jira Creation
27-
uses: ./whitesource-actions/snyk-issue
28-
with:
29-
snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
30-
snyk_token: ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN_PUBLIC_REPO }}
31-
jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
32-
env:
33-
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1+
#name: Snyk Issue
2+
#
3+
#on:
4+
# schedule:
5+
# - cron: '* */12 * * *'
6+
#
7+
#permissions:
8+
# contents: read
9+
# issues: write
10+
# pull-requests: write
11+
#
12+
#concurrency: snyk-issue
13+
#
14+
#jobs:
15+
# snyk:
16+
# runs-on: ubuntu-latest
17+
# steps:
18+
# - name: checkout action
19+
# uses: actions/checkout@v4
20+
# with:
21+
# repository: snowflakedb/whitesource-actions
22+
# token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
23+
# path: whitesource-actions
24+
# - name: set-env
25+
# run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
26+
# - name: Jira Creation
27+
# uses: ./whitesource-actions/snyk-issue
28+
# with:
29+
# snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
30+
# snyk_token: ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN_PUBLIC_REPO }}
31+
# jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
32+
# env:
33+
# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎.github/workflows/snyk-pr.yml‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,37 @@
1-
name: Snyk PR
2-
on:
3-
pull_request:
4-
branches:
5-
- main
6-
7-
permissions:
8-
contents: read
9-
issues: write
10-
pull-requests: write
11-
12-
jobs:
13-
snyk:
14-
runs-on: ubuntu-latest
15-
if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
16-
steps:
17-
- name: Checkout
18-
uses: actions/checkout@v3
19-
with:
20-
ref: ${{ github.event.pull_request.head.ref }}
21-
fetch-depth: 0
22-
23-
- name: Checkout Action
24-
uses: actions/checkout@v3
25-
with:
26-
repository: snowflakedb/whitesource-actions
27-
token: ${{ secrets.whitesource_action_token }}
28-
path: whitesource-actions
29-
30-
- name: Snyk Pull Request Scan Check
31-
uses: ./whitesource-actions/snyk-pr
32-
env:
33-
pr_title: ${{ github.event.pull_request.title }}
34-
with:
35-
jira_token: ${{ secrets.jira_token_public_repo }}
36-
gh_token: ${{ secrets.github_token }}
37-
amend: false
1+
#name: Snyk PR
2+
#on:
3+
# pull_request:
4+
# branches:
5+
# - main
6+
#
7+
#permissions:
8+
# contents: read
9+
# issues: write
10+
# pull-requests: write
11+
#
12+
#jobs:
13+
# snyk:
14+
# runs-on: ubuntu-latest
15+
# if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }}
16+
# steps:
17+
# - name: Checkout
18+
# uses: actions/checkout@v3
19+
# with:
20+
# ref: ${{ github.event.pull_request.head.ref }}
21+
# fetch-depth: 0
22+
#
23+
# - name: Checkout Action
24+
# uses: actions/checkout@v3
25+
# with:
26+
# repository: snowflakedb/whitesource-actions
27+
# token: ${{ secrets.whitesource_action_token }}
28+
# path: whitesource-actions
29+
#
30+
# - name: Snyk Pull Request Scan Check
31+
# uses: ./whitesource-actions/snyk-pr
32+
# env:
33+
# pr_title: ${{ github.event.pull_request.title }}
34+
# with:
35+
# jira_token: ${{ secrets.jira_token_public_repo }}
36+
# gh_token: ${{ secrets.github_token }}
37+
# amend: false

‎setup.cfg‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ boto =
8282
boto3>=1.24
8383
botocore>=1.24
8484
development =
85+
boto3
8586
Cython
8687
coverage
8788
more-itertools

‎test/integ/conftest.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,35 @@ def print_help() -> None:
175175
)
176176

177177

178+
# START changes for LocalStack
179+
180+
181+
@pytest.fixture(scope="session", autouse=True)
182+
def create_external_volume():
183+
import boto3
184+
185+
# create S3 bucket in LocalStack
186+
s3_client = boto3.client(
187+
"s3",
188+
endpoint_url="http://localhost:4566",
189+
aws_access_key_id="test",
190+
aws_secret_access_key="test",
191+
)
192+
s3_client.create_bucket(Bucket="test")
193+
194+
# create a local external volume which is required by some of the tests
195+
query = (
196+
"CREATE EXTERNAL VOLUME IF NOT EXISTS python_connector_iceberg_exvol STORAGE_LOCATIONS = "
197+
"((NAME = 'sl1' STORAGE_PROVIDER='S3' STORAGE_BASE_URL='s3://test' "
198+
"STORAGE_AWS_ROLE_ARN='arn:aws:iam::000000000000:role/test'))"
199+
)
200+
connection = create_connection("default")
201+
connection.cursor().execute(query)
202+
203+
204+
# END changes for LocalStack
205+
206+
178207
@pytest.fixture(scope="session")
179208
def is_public_test() -> bool:
180209
return is_public_testaccount()

‎test/integ/test_arrow_result.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,13 @@
167167
STRUCTURED_TYPE_ENVIRONMENTS = {"SFCTEST0_AWS_US_WEST_2", "SNOWPARK_PYTHON_TEST"}
168168
ICEBERG_ENVIRONMENTS = {"SFCTEST0_AWS_US_WEST_2"}
169169

170+
# ICEBERG_SUPPORTED = CLOUD in ICEBERG_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
171+
# STRUCTURED_TYPES_SUPPORTED = (
172+
# CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev"
173+
# )
174+
# Note whummer (2024-12-29): setting this value to False, as otherwise invalid tests are being generated,
175+
# using, e.g., `CREATE TABLE(c1 ARRAY(FLOAT))` which is invalid in standard case (verified against real SF)
176+
STRUCTURED_TYPES_SUPPORTED = False
170177

171178
# Generate all valid test cases. By using pytest.param with an id you can
172179
# run a specific test case easier like so:
@@ -1065,6 +1072,7 @@ def test_select_date(conn_cnx):
10651072
finish(conn_cnx, table)
10661073

10671074

1075+
@pytest.mark.skip(reason="LocalStack - currently incompatible with Snowflake emulator")
10681076
@pytest.mark.parametrize("scale", range(10))
10691077
@pytest.mark.parametrize("type", ["timestampntz", "timestampltz", "timestamptz"])
10701078
def test_select_timestamp_with_scale(conn_cnx, scale, type):

‎test/integ/test_cursor.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,6 +1596,9 @@ def test_resultbatch(
15961596
assert total_rows == rowcount
15971597

15981598

1599+
@pytest.mark.skip(
1600+
reason="Multi-batches and large result sets currently not yet working in LocalStack"
1601+
)
15991602
@pytest.mark.skipolddriver(reason="new feature in v2.5.0")
16001603
@pytest.mark.parametrize(
16011604
"result_format,patch_path",

0 commit comments

Comments
 (0)