Allow not specifying parameters names when using placeholder templater #13134
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
############################# | |
## GitHub Actions CI Tests ## | |
############################# | |
# | |
# This can be kicked off manually in the Actions tab of GitHub | |
# It will also run nightly at 2pm | |
# It will run on any pull request, except non-code changes | |
# (images, markdown files, ) | |
# | |
name: CI | |
on: | |
workflow_dispatch: | |
schedule: | |
# 2am each night | |
- cron: '00 2 * * *' | |
pull_request: | |
push: | |
branches: | |
- main | |
merge_group: | |
# Merge Queue checks requested. This feature is still in beta | |
# from Github and so may need updating later. | |
# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#merge_group | |
types: [checks_requested] | |
jobs: | |
linting: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
jobs: [ 'linting', 'doclinting', 'docbuild', 'yamllint' ] | |
name: ${{ matrix.jobs }} tests | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: pip install tox | |
- name: Run the tests | |
run: tox -e ${{ matrix.jobs }} | |
# Test with coverage tracking on most recent python (py11). | |
python-version-tests-cov: | |
name: Python 3.11 Tests | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-python.yml | |
with: | |
python-version: "3.11" | |
coverage: true | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
# Test without coverage tracking on older python versions. | |
# This saves time, as testing without coverage tracking is faster. | |
python-version-tests-nocov: | |
name: Python ${{ matrix.python-version }} Tests | |
strategy: | |
matrix: | |
python-version: [ '3.7', '3.8', '3.9', '3.10' ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-python.yml | |
with: | |
python-version: ${{ matrix.python-version }} | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
dbt-tests-cov: | |
name: dbt ${{ matrix.dbt-version }} Plugin Tests | |
strategy: | |
matrix: | |
dbt-version: [ "dbt110", "dbt130", "dbt140", "dbt150", "dbt160" ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-dbt.yml | |
with: | |
python-version: "3.10" | |
dbt-version: "${{ matrix.dbt-version }}" | |
coverage: true | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
dbt-tests-nocov: | |
name: dbt ${{ matrix.dbt-version }} Plugin Tests | |
strategy: | |
matrix: | |
dbt-version: [ "dbt120" ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-dbt.yml | |
with: | |
python-version: "3.9" | |
dbt-version: "${{ matrix.dbt-version }}" | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
# This runs the bulk of the dialect _parsing_ tests. | |
# | |
# It's run as a separate job as takes longer than the CI jobs and allows | |
# them to be rerun separately if GitHub Actions or Coverage is experiencing | |
# issues. | |
dialects_parse: | |
name: Dialect parsing | |
strategy: | |
matrix: | |
python-version: [ '3.11' ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-python.yml | |
with: | |
python-version: "3.11" | |
marks: "parse_suite" | |
# We test coverage here for some parsing routines. | |
coverage: true | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
# This lints all our dialect fixtures to check rules can handle a variety | |
# of SQL and don't error out badly. | |
# | |
# It's run as a separate job as takes longer than the CI jobs and allows | |
# them to be rerun separately if GitHub Actions or Coverage is experiencing | |
# issues. | |
dialects_fix: | |
name: Dialect fixing | |
strategy: | |
matrix: | |
python-version: [ '3.11' ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-python.yml | |
with: | |
python-version: "3.11" | |
marks: "fix_suite" | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
# This lints all our rules fixtures to check rules. | |
# | |
# It's run as a separate job as takes longer than the CI jobs and allows | |
# them to be rerun separately if GitHub Actions or Coverage is experiencing | |
# issues. | |
rules: | |
name: Rule yaml test cases | |
strategy: | |
matrix: | |
python-version: [ '3.11' ] | |
permissions: | |
contents: read | |
pull-requests: write | |
uses: ./.github/workflows/ci-test-python.yml | |
with: | |
python-version: "3.11" | |
marks: "rules_suite" | |
coverage: true | |
secrets: | |
gh_token: ${{ secrets.github_token }} | |
other-tests: | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
jobs: [ 'bench', 'mypy', 'doctests' ] | |
name: ${{ matrix.jobs }} tests | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: pip install tox | |
- name: Run the tests | |
env: | |
SQLFLUFF_BENCHMARK_API_KEY: ${{ secrets.SQLFLUFF_BENCHMARK_API_KEY }} | |
run: | | |
tox -e ${{ matrix.jobs }} | |
ymlchecks: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python 3.11 | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -r requirements.txt -r requirements_dev.txt | |
python setup.py develop | |
- name: Generate the YAML files | |
run: | | |
python test/generate_parse_fixture_yml.py | |
- name: Test the generated YAML files | |
run: | | |
if [ -n "$(git status --porcelain)" ]; then | |
git diff | |
echo "Generated YAML files do not match branch." | |
echo "Please run the following command to generate these:" | |
echo " python test/generate_parse_fixture_yml.py" | |
exit 1 | |
fi | |
examples: | |
runs-on: ubuntu-latest | |
name: example tests | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
- name: Install dependencies | |
run: | | |
pip install -e . | |
pip install tqdm | |
- name: Test the example files | |
run: | | |
for file in examples/* | |
do | |
echo "Running $file" | |
python "$file" | |
done | |
python-windows-tests: | |
runs-on: windows-latest | |
name: Python 3.10 Windows tests | |
steps: | |
- name: Set git to use LF | |
run: | | |
git config --global core.autocrlf false | |
git config --global core.eol lf | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.10' | |
- name: List Env | |
shell: bash | |
run: | | |
env | sort | |
- name: Install dependencies | |
shell: bash | |
run: pip install tox | |
- name: Run the tests | |
shell: bash | |
# Set python temp dir in working dir as on GitHub Actions Windows | |
# machine often has system temp dir (which tox uses) on C drive and | |
# working dir on D drive which causes problems. | |
run: | | |
mkdir temp_pytest | |
python -m tox -e winpy -- --cov=sqlfluff -n 2 test -m "not integration" | |
- name: Upload coverage data (github) | |
uses: actions/upload-artifact@v3 | |
with: | |
name: coverage-data | |
path: ".coverage.*" | |
if-no-files-found: ignore | |
python-windows-dbt-tests: | |
runs-on: windows-latest | |
name: DBT Plugin Python 3.10 Windows tests | |
steps: | |
- name: Start PostgreSQL on Windows | |
run: | | |
$pgService = Get-Service -Name postgresql* | |
Set-Service -InputObject $pgService -Status running -StartupType automatic | |
Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru | |
- name: Set postgres user password | |
run: | | |
& $env:PGBIN\psql --command="ALTER USER postgres PASSWORD 'password';" --command="\du" | |
- name: Set git to use LF | |
run: | | |
git config --global core.autocrlf false | |
git config --global core.eol lf | |
- uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.10' | |
- name: Install dependencies | |
shell: bash | |
run: pip install tox | |
- name: Run the tests | |
shell: bash | |
# Do not set explicitly temp dir for dbt as causes problems | |
# None of these test need temp dir set | |
run: | | |
python -m tox -e dbt150-winpy -- plugins/sqlfluff-templater-dbt | |
pip-test-pull-request: | |
# Test that using pip install works as we've missed | |
# some dependencies in the past - see #1842 | |
runs-on: ubuntu-latest | |
if: github.event_name == 'pull_request' | |
name: pip install tests | |
steps: | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: '3.11' | |
- uses: actions/checkout@v3 | |
- name: Install dependencies | |
run: | | |
pip install . | |
- name: Run the version test | |
run: | | |
sqlfluff --version | |
- name: Run a simple select parse test via stdin | |
run: | | |
echo "select 1" | sqlfluff parse --dialect=ansi - | |
- name: Run a simple select lint test via stdin | |
run: | | |
echo "select 1" | sqlfluff lint --dialect=ansi - | |
- name: Run a simple select parse test via file | |
run: | | |
sqlfluff parse --dialect=ansi <(echo "select 1") | |
- name: Run a simple select lint test via file | |
run: | | |
sqlfluff lint --dialect=ansi <(echo "select 1") | |
coveralls_finish: | |
name: Finalise coveralls. | |
needs: [python-version-tests-cov, dbt-tests-cov, python-windows-tests, dialects_parse, rules] | |
runs-on: ubuntu-latest | |
steps: | |
- name: Coveralls Finished | |
uses: coverallsapp/github-action@master | |
with: | |
github-token: ${{ secrets.github_token }} | |
parallel-finished: true | |
coverage_check: | |
name: Combine & check 100% coverage. | |
runs-on: ubuntu-latest | |
needs: [python-version-tests-cov, dbt-tests-cov, python-windows-tests, dialects_parse, rules] | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v4 | |
with: | |
python-version: "3.10" | |
- run: python -m pip install --upgrade coverage[toml] | |
- name: Download coverage data. | |
uses: actions/download-artifact@v3 | |
with: | |
name: coverage-data | |
- name: Combine coverage & fail if it's <100%. | |
id: report_coverage | |
# NOTE: Setting the pipefail option here means that even when | |
# piping the output to `tee`, we still get the exit code of the | |
# `coverage report` command. | |
run: | | |
set -o pipefail | |
python -m coverage combine | |
python -m coverage html --skip-covered --skip-empty | |
python -m coverage report --fail-under=100 --skip-covered --skip-empty -m | tee coverage-report.txt | |
- name: Upload HTML report if check failed. | |
uses: actions/upload-artifact@v3 | |
with: | |
name: html-report | |
path: htmlcov | |
if: failure() && github.event_name == 'pull_request' | |
- name: Update PR comment with coverage report. | |
uses: actions/github-script@v6 | |
# Run even if the coverage step failed. | |
if: always() && github.event_name == 'pull_request' | |
with: | |
script: | | |
// First list the existing comments | |
const trigger_str = 'Coverage Results'; | |
console.log("Getting existing comments..."); | |
const issue_number = context.issue.number; | |
console.log("Issue number: " + issue_number); | |
const comments = await github.paginate( | |
github.rest.issues.listComments, | |
{ | |
owner: 'sqlfluff', | |
repo: 'sqlfluff', | |
issue_number: issue_number | |
} | |
); | |
let comment_id = null; | |
console.log("Got %d comments", comments.length); | |
comments.forEach(comment => { | |
if (comment.body.indexOf(trigger_str) >= 0) { | |
console.log("Found target comment ID: %d", comment.id); | |
comment_id = comment.id; | |
} else { | |
console.log("Comment ID %d not valid with body:\n%s.", comment.id, comment.body); | |
} | |
}); | |
const previous_outcome = "${{ steps.report_coverage.outcome }}"; | |
console.log("Previous coverage step outcome: %s", previous_outcome); | |
if (previous_outcome == "success") { | |
status_emoji = "✅"; | |
} else { | |
status_emoji = "⚠️"; | |
} | |
const { promises: fs } = require('fs'); | |
const content = await fs.readFile('coverage-report.txt', 'utf8'); | |
body = "# " + trigger_str + " " + status_emoji + "\n```\n" + content + "\n```\n"; | |
try { | |
if (comment_id > 0) { | |
console.log("Updating comment id: %d", comment_id); | |
await github.rest.issues.updateComment({ | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
comment_id: comment_id, | |
body: body | |
}); | |
} else { | |
console.log("No existing comment matched, creating a new one..."); | |
await github.rest.issues.createComment({ | |
issue_number: context.issue.number, | |
owner: context.repo.owner, | |
repo: context.repo.repo, | |
body: body | |
}); | |
} | |
} catch(err) { | |
// NOTE: This is most likely if the PR is opened from a branch | |
// of a forked repo. | |
console.log("Failed to create/update comment. Skipping..."); | |
} |