diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index ab98dd3702..bec033cd05 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -51,7 +51,8 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] +# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 - name: Set up Python @@ -72,15 +73,16 @@ jobs: os: - image: ubuntu-20.04 id: manylinux_x86_64 - - image: ubuntu-20.04 - id: manylinux_aarch64 - - image: windows-2019 - id: win_amd64 - - image: macos-latest - id: macosx_x86_64 - - image: macos-latest - id: macosx_arm64 - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] +# - image: ubuntu-20.04 +# id: manylinux_aarch64 +# - image: windows-2019 +# id: win_amd64 +# - image: macos-latest +# id: macosx_x86_64 +# - image: macos-latest +# id: macosx_arm64 +# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11"] name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }} runs-on: ${{ matrix.os.image }} steps: @@ -114,6 +116,7 @@ jobs: test: name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} needs: build + timeout-minutes: 30 runs-on: ${{ matrix.os.image_name }} strategy: fail-fast: false @@ -121,12 +124,14 @@ jobs: os: - image_name: ubuntu-latest download_name: manylinux_x86_64 - - image_name: macos-latest - download_name: macosx_x86_64 - - image_name: windows-2019 - download_name: win_amd64 - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - cloud-provider: [aws, azure, gcp] +# - image_name: macos-latest +# download_name: macosx_x86_64 +# - image_name: windows-2019 +# download_name: win_amd64 +# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.11"] +# cloud-provider: [aws, azure, gcp] + cloud-provider: [aws] steps: - uses: actions/checkout@v4 - name: Set up Python @@ -135,13 +140,25 @@ jobs: python-version: ${{ matrix.python-version }} - name: Display Python version run: python -c "import sys; print(sys.version)" - - name: Setup parameters file - shell: bash + + - name: Start up LocalStack Snowflake env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} + LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py + image_name=localstack/snowflake + # image_name=whummer/snowflake-tmp + docker pull $image_name & + pip install localstack + IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace' localstack start -d + localstack wait + +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - name: Download wheel(s) uses: actions/download-artifact@v4 with: @@ -155,13 +172,15 @@ jobs: - name: Install tox run: python -m pip install tox>=4 - name: Run tests - run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'` +# run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,integ,pandas,sso}-ci | sed 's/ /,/g'` + run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,integ}-ci | sed 's/ /,/g'` env: PYTHON_VERSION: ${{ matrix.python-version }} cloud_provider: ${{ matrix.cloud-provider }} PYTEST_ADDOPTS: --color=yes --tb=short TOX_PARALLEL_NO_SPINNER: 1 shell: bash + timeout-minutes: 15 - name: Combine coverages run: python -m tox run -e coverage --skip-missing-interpreters false shell: bash @@ -173,213 +192,213 @@ jobs: .tox/.coverage .tox/coverage.xml - test-olddriver: - name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: lint - runs-on: ${{ matrix.os.image_name }} - strategy: - fail-fast: false - matrix: - os: - - image_name: ubuntu-latest - download_name: linux - python-version: [3.8] - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Upgrade setuptools, pip and wheel - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Run tests - run: python -m tox run -e olddriver - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - shell: bash + - name: Print LocalStack logs + run: localstack logs + if: ${{ success() || failure() }} - test-noarrowextension: - name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: lint - runs-on: ${{ matrix.os.image_name }} - strategy: - fail-fast: false - matrix: - os: - - image_name: ubuntu-latest - download_name: linux - python-version: [3.8] - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Upgrade setuptools, pip and wheel - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Run tests - run: python -m tox run -e noarrowextension - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - shell: bash +# test-olddriver: +# name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: lint +# runs-on: ${{ matrix.os.image_name }} +# strategy: +# fail-fast: false +# matrix: +# os: +# - image_name: ubuntu-latest +# download_name: linux +# python-version: [3.8] +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v3 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Upgrade setuptools, pip and wheel +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Run tests +# run: python -m tox run -e olddriver +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# shell: bash - test-fips: - name: Test FIPS linux-3.8-${{ matrix.cloud-provider }} - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Download wheel(s) - uses: actions/download-artifact@v4 - with: - name: manylinux_x86_64_py3.8 - path: dist - - name: Show wheels downloaded - run: ls -lh dist - shell: bash - - name: Run tests - run: ./ci/test_fips_docker.sh - env: - PYTHON_VERSION: 3.8 - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - TOX_PARALLEL_NO_SPINNER: 1 - shell: bash - - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }} - path: | - .coverage - coverage.xml +# test-noarrowextension: +# name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: lint +# runs-on: ${{ matrix.os.image_name }} +# strategy: +# fail-fast: false +# matrix: +# os: +# - image_name: ubuntu-latest +# download_name: linux +# python-version: [3.8] +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v3 +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Upgrade setuptools, pip and wheel +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Run tests +# run: python -m tox run -e noarrowextension +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# shell: bash - test-lambda: - name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - cloud-provider: [aws] - steps: - - name: Set shortver - run: echo "shortver=${longver//./}" >> $GITHUB_ENV - env: - longver: ${{ matrix.python-version }} - shell: bash - - uses: actions/checkout@v4 - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Download wheel(s) - uses: actions/download-artifact@v4 - with: - name: manylinux_x86_64_py${{ matrix.python-version }} - path: dist - - name: Show wheels downloaded - run: ls -lh dist - shell: bash - - name: Run tests - run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - TOX_PARALLEL_NO_SPINNER: 1 - shell: bash - - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - path: | - .coverage.py${{ env.shortver }}-lambda-ci - junit.py${{ env.shortver }}-lambda-ci-dev.xml +# test-fips: +# name: Test FIPS linux-3.8-${{ matrix.cloud-provider }} +# needs: build +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# cloud-provider: [aws] +# steps: +# - uses: actions/checkout@v3 +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Download wheel(s) +# uses: actions/download-artifact@v3 +# with: +# name: manylinux_x86_64_py3.8 +# path: dist +# - name: Show wheels downloaded +# run: ls -lh dist +# shell: bash +# - name: Run tests +# run: ./ci/test_fips_docker.sh +# env: +# PYTHON_VERSION: 3.8 +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# TOX_PARALLEL_NO_SPINNER: 1 +# shell: bash +# - uses: actions/upload-artifact@v3 +# with: +# name: coverage_linux-fips-3.8-${{ matrix.cloud-provider }} +# path: | +# .coverage +# coverage.xml - combine-coverage: - if: ${{ success() || failure() }} - name: Combine coverage - needs: [lint, test, test-fips, test-lambda] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Upgrade setuptools and pip - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Collect all coverages to one dir - run: | - python -c ' - from pathlib import Path - import shutil +# test-lambda: +# name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# needs: build +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +## python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] +# python-version: ["3.11"] +# cloud-provider: [aws] +# steps: +# - name: Set shortver +# run: echo "shortver=${longver//./}" >> $GITHUB_ENV +# env: +# longver: ${{ matrix.python-version }} +# shell: bash +# - uses: actions/checkout@v3 +# - name: Setup parameters file +# shell: bash +# env: +# PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} +# run: | +# gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ +# .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py +# - name: Download wheel(s) +# uses: actions/download-artifact@v3 +# with: +# name: manylinux_x86_64_py${{ matrix.python-version }} +# path: dist +# - name: Show wheels downloaded +# run: ls -lh dist +# shell: bash +# - name: Run tests +# run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION} +# env: +# PYTHON_VERSION: ${{ matrix.python-version }} +# cloud_provider: ${{ matrix.cloud-provider }} +# PYTEST_ADDOPTS: --color=yes --tb=short +# TOX_PARALLEL_NO_SPINNER: 1 +# shell: bash +# - uses: actions/upload-artifact@v3 +# with: +# name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }} +# path: | +# .coverage.py${{ env.shortver }}-lambda-ci +# junit.py${{ env.shortver }}-lambda-ci-dev.xml - src_dir = Path("artifacts") - dst_dir = Path(".") / ".tox" - dst_dir.mkdir() - for src_file in src_dir.glob("*/.coverage"): - dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) - print("{} copy to {}".format(src_file, dst_file)) - shutil.copy(str(src_file), str(dst_file))' - - name: Combine coverages - run: python -m tox run -e coverage - - name: Publish html coverage - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: overall_cov_html - path: .tox/htmlcov - - name: Publish xml coverage - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: overall_cov_xml - path: .tox/coverage.xml - - uses: codecov/codecov-action@v4 - with: - files: .tox/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} +# combine-coverage: +# if: ${{ success() || failure() }} +# name: Combine coverage +# needs: [lint, test, test-fips, test-lambda] +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/download-artifact@v3 +# with: +# path: artifacts +# - name: Set up Python +# uses: actions/setup-python@v4 +# with: +# python-version: '3.8' +# - name: Display Python version +# run: python -c "import sys; print(sys.version)" +# - name: Upgrade setuptools and pip +# run: python -m pip install -U setuptools pip wheel +# - name: Install tox +# run: python -m pip install tox>=4 +# - name: Collect all coverages to one dir +# run: | +# python -c ' +# from pathlib import Path +# import shutil +# +# src_dir = Path("artifacts") +# dst_dir = Path(".") / ".tox" +# dst_dir.mkdir() +# for src_file in src_dir.glob("*/.coverage"): +# dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) +# print("{} copy to {}".format(src_file, dst_file)) +# shutil.copy(str(src_file), str(dst_file))' +# - name: Combine coverages +# run: python -m tox run -e coverage +## - name: Publish html coverage +## uses: actions/upload-artifact@v3 +## with: +## name: overall_cov_html +## path: .tox/htmlcov +## - name: Publish xml coverage +## uses: actions/upload-artifact@v3 +## with: +## name: overall_cov_xml +## path: .tox/coverage.xml +# - uses: codecov/codecov-action@v3 +# with: +# files: .tox/coverage.xml diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 2e197168c8..6bb72b8906 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -1,20 +1,20 @@ -name: Changelog Check - -on: - pull_request: - types: [opened, synchronize, labeled, unlabeled] - branches: - - main - -jobs: - check_change_log: - runs-on: ubuntu-latest - if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Ensure DESCRIPTION.md is updated - run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md" +#name: Changelog Check +# +#on: +# pull_request: +# types: [opened, synchronize, labeled, unlabeled] +# branches: +# - main +# +#jobs: +# check_change_log: +# runs-on: ubuntu-latest +# if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}} +# steps: +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# +# - name: Ensure DESCRIPTION.md is updated +# run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "DESCRIPTION.md" diff --git a/.github/workflows/cla_bot.yml b/.github/workflows/cla_bot.yml index af8e77f90f..119197c2ad 100644 --- a/.github/workflows/cla_bot.yml +++ b/.github/workflows/cla_bot.yml @@ -1,29 +1,29 @@ -name: "CLA Assistant" -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened,closed,synchronize] - -jobs: - CLAssistant: - runs-on: ubuntu-latest - permissions: - actions: write - contents: write - pull-requests: write - statuses: write - steps: - - name: "CLA Assistant" - if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' - uses: contributor-assistant/github-action/@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }} - with: - path-to-signatures: 'signatures/version1.json' - path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md' - branch: 'main' - allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa' - remote-organization-name: 'snowflakedb' - remote-repository-name: 'cla-db' +#name: "CLA Assistant" +#on: +# issue_comment: +# types: [created] +# pull_request_target: +# types: [opened,closed,synchronize] +# +#jobs: +# CLAssistant: +# runs-on: ubuntu-latest +# permissions: +# actions: write +# contents: write +# pull-requests: write +# statuses: write +# steps: +# - name: "CLA Assistant" +# if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target' +# uses: contributor-assistant/github-action/@master +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }} +# with: +# path-to-signatures: 'signatures/version1.json' +# path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md' +# branch: 'main' +# allowlist: 'dependabot[bot],github-actions,Jenkins User,sfc-gh-snyk-sca-sa' +# remote-organization-name: 'snowflakedb' +# remote-repository-name: 'cla-db' diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index c417eb113d..b0a3f886d7 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -1,15 +1,15 @@ ---- -name: Run semgrep checks - -on: - pull_request: - branches: [main] - -permissions: - contents: read - -jobs: - run-semgrep-reusable-workflow: - uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main - secrets: - token: ${{ secrets.SEMGREP_APP_TOKEN }} +#--- +#name: Run semgrep checks +# +#on: +# pull_request: +# branches: [main] +# +#permissions: +# contents: read +# +#jobs: +# run-semgrep-reusable-workflow: +# uses: snowflakedb/reusable-workflows/.github/workflows/semgrep-v2.yml@main +# secrets: +# token: ${{ secrets.SEMGREP_APP_TOKEN }} diff --git a/.github/workflows/snyk-issue.yml b/.github/workflows/snyk-issue.yml index 486d0be5b3..c701c5ced5 100644 --- a/.github/workflows/snyk-issue.yml +++ b/.github/workflows/snyk-issue.yml @@ -1,33 +1,33 @@ -name: Snyk Issue - -on: - schedule: - - cron: '* */12 * * *' - -permissions: - contents: read - issues: write - pull-requests: write - -concurrency: snyk-issue - -jobs: - snyk: - runs-on: ubuntu-latest - steps: - - name: Checkout Action - uses: actions/checkout@v3 - with: - repository: snowflakedb/whitesource-actions - token: ${{ secrets.whitesource_action_token }} - path: whitesource-actions - - name: Set Env - run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV - - name: Jira Creation - uses: ./whitesource-actions/snyk-issue - with: - snyk_org: ${{ secrets.snyk_org_id_public_repo }} - snyk_token: ${{ secrets.snyk_github_integration_token_public_repo }} - jira_token: ${{ secrets.jira_token_public_repo }} - env: - gh_token: ${{ secrets.github_token }} +#name: Snyk Issue +# +#on: +# schedule: +# - cron: '* */12 * * *' +# +#permissions: +# contents: read +# issues: write +# pull-requests: write +# +#concurrency: snyk-issue +# +#jobs: +# snyk: +# runs-on: ubuntu-latest +# steps: +# - name: Checkout Action +# uses: actions/checkout@v3 +# with: +# repository: snowflakedb/whitesource-actions +# token: ${{ secrets.whitesource_action_token }} +# path: whitesource-actions +# - name: Set Env +# run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV +# - name: Jira Creation +# uses: ./whitesource-actions/snyk-issue +# with: +# snyk_org: ${{ secrets.snyk_org_id_public_repo }} +# snyk_token: ${{ secrets.snyk_github_integration_token_public_repo }} +# jira_token: ${{ secrets.jira_token_public_repo }} +# env: +# gh_token: ${{ secrets.github_token }} diff --git a/.github/workflows/snyk-pr.yml b/.github/workflows/snyk-pr.yml index b951af65f4..3a00f8624d 100644 --- a/.github/workflows/snyk-pr.yml +++ b/.github/workflows/snyk-pr.yml @@ -1,37 +1,37 @@ -name: Snyk PR -on: - pull_request: - branches: - - main - -permissions: - contents: read - issues: write - pull-requests: write - -jobs: - snyk: - runs-on: ubuntu-latest - if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.ref }} - fetch-depth: 0 - - - name: Checkout Action - uses: actions/checkout@v3 - with: - repository: snowflakedb/whitesource-actions - token: ${{ secrets.whitesource_action_token }} - path: whitesource-actions - - - name: Snyk Pull Request Scan Check - uses: ./whitesource-actions/snyk-pr - env: - pr_title: ${{ github.event.pull_request.title }} - with: - jira_token: ${{ secrets.jira_token_public_repo }} - gh_token: ${{ secrets.github_token }} - amend: false +#name: Snyk PR +#on: +# pull_request: +# branches: +# - main +# +#permissions: +# contents: read +# issues: write +# pull-requests: write +# +#jobs: +# snyk: +# runs-on: ubuntu-latest +# if: ${{ github.event.pull_request.user.login == 'sfc-gh-snyk-sca-sa' }} +# steps: +# - name: Checkout +# uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.ref }} +# fetch-depth: 0 +# +# - name: Checkout Action +# uses: actions/checkout@v3 +# with: +# repository: snowflakedb/whitesource-actions +# token: ${{ secrets.whitesource_action_token }} +# path: whitesource-actions +# +# - name: Snyk Pull Request Scan Check +# uses: ./whitesource-actions/snyk-pr +# env: +# pr_title: ${{ github.event.pull_request.title }} +# with: +# jira_token: ${{ secrets.jira_token_public_repo }} +# gh_token: ${{ secrets.github_token }} +# amend: false diff --git a/setup.cfg b/setup.cfg index 41974aac74..88a86ec20a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -79,6 +79,7 @@ console_scripts = [options.extras_require] development = + boto3 Cython coverage more-itertools diff --git a/test/integ/conftest.py b/test/integ/conftest.py index 0f112ec305..67b24ad6c4 100644 --- a/test/integ/conftest.py +++ b/test/integ/conftest.py @@ -83,6 +83,35 @@ def print_help() -> None: ) +# START changes for LocalStack + + +@pytest.fixture(scope="session", autouse=True) +def create_external_volume(): + import boto3 + + # create S3 bucket in LocalStack + s3_client = boto3.client( + "s3", + endpoint_url="http://localhost:4566", + aws_access_key_id="test", + aws_secret_access_key="test", + ) + s3_client.create_bucket(Bucket="test") + + # create a local external volume which is required by some of the tests + query = ( + "CREATE EXTERNAL VOLUME IF NOT EXISTS python_connector_iceberg_exvol STORAGE_LOCATIONS = " + "((NAME = 'sl1' STORAGE_PROVIDER='S3' STORAGE_BASE_URL='s3://test' " + "STORAGE_AWS_ROLE_ARN='arn:aws:iam::000000000000:role/test'))" + ) + connection = create_connection("default") + connection.cursor().execute(query) + + +# END changes for LocalStack + + @pytest.fixture(scope="session") def is_public_test() -> bool: return is_public_testaccount() diff --git a/test/integ/test_arrow_result.py b/test/integ/test_arrow_result.py index d8118617d1..f18cc8e734 100644 --- a/test/integ/test_arrow_result.py +++ b/test/integ/test_arrow_result.py @@ -172,9 +172,12 @@ RUNNING_ON_GH = os.getenv("GITHUB_ACTIONS") == "true" ICEBERG_SUPPORTED = CLOUD in ICEBERG_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev" -STRUCTURED_TYPES_SUPPORTED = ( - CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev" -) +# STRUCTURED_TYPES_SUPPORTED = ( +# CLOUD in STRUCTRED_TYPE_ENVIRONMENTS and RUNNING_ON_GH or CLOUD == "dev" +# ) +# Note whummer (2024-12-29): setting this value to False, as otherwise invalid tests are being generated, +# using, e.g., `CREATE TABLE(c1 ARRAY(FLOAT))` which is invalid in standard case (verified against real SF) +STRUCTURED_TYPES_SUPPORTED = False # Generate all valid test cases. By using pytest.param with an id you can # run a specific test case easier like so: @@ -937,6 +940,7 @@ def test_select_date(conn_cnx): finish(conn_cnx, table) +@pytest.mark.skip(reason="LocalStack - currently incompatible with Snowflake emulator") @pytest.mark.parametrize("scale", range(10)) @pytest.mark.parametrize("type", ["timestampntz", "timestampltz", "timestamptz"]) def test_select_timestamp_with_scale(conn_cnx, scale, type): diff --git a/test/integ/test_cursor.py b/test/integ/test_cursor.py index 384e5e95a1..f475f874b4 100644 --- a/test/integ/test_cursor.py +++ b/test/integ/test_cursor.py @@ -1556,6 +1556,9 @@ def test_resultbatch( assert total_rows == rowcount +@pytest.mark.skip( + reason="Multi-batches and large result sets currently not yet working in LocalStack" +) @pytest.mark.skipolddriver(reason="new feature in v2.5.0") @pytest.mark.parametrize( "result_format,patch_path", diff --git a/test/parameters.py b/test/parameters.py new file mode 100644 index 0000000000..f55d3a3369 --- /dev/null +++ b/test/parameters.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2012-2023 Snowflake Computing Inc. All rights reserved. +# + +CONNECTION_PARAMETERS = { + "account": "test", + "user": "test", + "password": "test", + "database": "test", + "schema": "public", + "host": "snowflake.localhost.localstack.cloud", +} diff --git a/tox.ini b/tox.ini index 6faca8c0d8..680bb36862 100644 --- a/tox.ini +++ b/tox.ini @@ -131,7 +131,7 @@ deps = flake8 commands = flake8 {posargs} [testenv:fix_lint] -basepython = python3.8 +# basepython = python3.8 description = format the code base to adhere to our styles, and complain about what we cannot do automatically passenv = PROGRAMDATA