From 440c9d069daa473bf285651d8aa6bf69b75a52b2 Mon Sep 17 00:00:00 2001 From: Xilin Wang Date: Wed, 8 Jun 2022 10:06:00 -0400 Subject: [PATCH] merge into master --- .github/workflows/ci-integration.yml | 674 +++++++++++++++++++++++++++ .github/workflows/ci-unit.yml | 254 ++++++++++ .github/workflows/ok-to-test.yml | 42 ++ 3 files changed, 970 insertions(+) create mode 100644 .github/workflows/ci-integration.yml create mode 100644 .github/workflows/ci-unit.yml create mode 100644 .github/workflows/ok-to-test.yml diff --git a/.github/workflows/ci-integration.yml b/.github/workflows/ci-integration.yml new file mode 100644 index 000000000..e5fce6460 --- /dev/null +++ b/.github/workflows/ci-integration.yml @@ -0,0 +1,674 @@ +# Run secret-dependent integration tests only after /ok-to-test approval +on: + pull_request: + repository_dispatch: + types: [ok-to-test-command] + +name: Integration tests + +jobs: + # Branch-based pull request + integration-trusted-linux: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + + - name: Branch based PR checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Config + run: | + conda create -q -n conda-env python=${{ matrix.python-version }} + + # for the tests that depend on git repos + git config --global user.email "someone@example.com" + git config --global user.name "Someone" + + - name: Install dependencies + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + # for some reason, conda does not resolve the env correctly and + # installs and old version of pygraphviz (1.3) which breaks in + # Python 3.7 + conda install -y "pygraphviz>=1.6" -c conda-forge + + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + python -c 'import ploomber.io' + + # install dev requirements + pip install .[dev] + + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + PLOOMBER_STATS_ENABLED: false + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + + # doc tests + export PY_IGNORE_IMPORTMISMATCH=1 + pytest src/ploomber --doctest-modules --ignore src/ploomber/resources + unset PY_IGNORE_IMPORTMISMATCH + + # unit tests + pytest tests/cli/test_cloud_execution.py --cov=ploomber + coveralls + + # Repo owner has commented /ok-to-test on a (fork-based) pull request + integration-fork-linux: + runs-on: ubuntu-latest + if: + github.event_name == 'repository_dispatch' && + github.event.client_payload.slash_command.sha != '' && + contains(github.event.client_payload.pull_request.head.sha, github.event.client_payload.slash_command.sha) + + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + steps: + + # Check out merge commit + - name: Fork based /ok-to-test checkout + uses: actions/checkout@v2 + with: + ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Config + run: | + conda create -q -n conda-env python=${{ matrix.python-version }} + + # for the tests that depend on git repos + git config --global user.email "someone@example.com" + git config --global user.name "Someone" + + - name: Install dependencies + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + # for some reason, conda does not resolve the env correctly and + # installs and old version of pygraphviz (1.3) which breaks in + # Python 3.7 + conda install -y "pygraphviz>=1.6" -c conda-forge + + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + python -c 'import ploomber.io' + + # install dev requirements + pip install .[dev] + + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + PLOOMBER_STATS_ENABLED: false + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + + # doc tests + export PY_IGNORE_IMPORTMISMATCH=1 + pytest src/ploomber --doctest-modules --ignore src/ploomber/resources + unset PY_IGNORE_IMPORTMISMATCH + + # unit tests + pytest tests/cli/test_cloud_execution.py --cov=ploomber + coveralls + + - run: | + echo "Integration tests... success! ;-)" + + # Update check run called "integration-fork" + - uses: actions/github-script@v5 + id: update-check-run + if: ${{ always() }} + env: + number: ${{ github.event.client_payload.pull_request.number }} + job: ${{ github.job }} + # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run + conclusion: ${{ job.status }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pull } = await github.rest.pulls.get({ + ...context.repo, + pull_number: process.env.number + }); + const ref = pull.head.sha; + + const { data: checks } = await github.rest.checks.listForRef({ + ...context.repo, + ref + }); + + const check = checks.check_runs.filter(c => c.name === process.env.job); + + const { data: result } = await github.rest.checks.update({ + ...context.repo, + check_run_id: check[0].id, + status: 'completed', + conclusion: process.env.conclusion + }); + + return result; + + + integration-trusted-macos: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Start PostgreSQL + run: | + brew services start postgresql + echo "Check PostgreSQL service is running" + i=10 + COMMAND='pg_isready' + while [ $i -gt 0 ]; do + echo "Check PostgreSQL service status" + eval $COMMAND && break + ((i--)) + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, all attempts exhausted" + exit 1 + fi + echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" + sleep 10 + done + + - name: Create PostgreSQL user + run: psql --command="CREATE USER postgres PASSWORD 'postgres'" postgres + + - name: Update PostgreSQL permissions + run: psql --command="ALTER DATABASE postgres OWNER TO postgres" postgres + + - name: Check PostgreSQL authentication + run: PGPASSWORD=postgres pg_isready --dbname postgres --username postgres + + - name: Branch based PR checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Install dependencies + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + conda install -y pygraphviz -c conda-forge + + # Test vanilla installation, make sure core modules are importable + # note: I needed to add "--ignore-installed certifi" because the + # installation started failing suddenly + pip install . --ignore-installed certifi + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests/cli/test_cloud_execution.py + + # Repo owner has commented /ok-to-test on a (fork-based) pull request + integration-fork-macos: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + if: + github.event_name == 'repository_dispatch' && + github.event.client_payload.slash_command.sha != '' && + contains(github.event.client_payload.pull_request.head.sha, github.event.client_payload.slash_command.sha) + steps: + - name: Start PostgreSQL + run: | + brew services start postgresql + echo "Check PostgreSQL service is running" + i=10 + COMMAND='pg_isready' + while [ $i -gt 0 ]; do + echo "Check PostgreSQL service status" + eval $COMMAND && break + ((i--)) + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, all attempts exhausted" + exit 1 + fi + echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" + sleep 10 + done + + - name: Create PostgreSQL user + run: psql --command="CREATE USER postgres PASSWORD 'postgres'" postgres + + - name: Update PostgreSQL permissions + run: psql --command="ALTER DATABASE postgres OWNER TO postgres" postgres + + - name: Check PostgreSQL authentication + run: PGPASSWORD=postgres pg_isready --dbname postgres --username postgres + + # Check out merge commit + - name: Fork based /ok-to-test checkout + uses: actions/checkout@v2 + with: + ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' + + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Install dependencies + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + conda install -y pygraphviz -c conda-forge + + # Test vanilla installation, make sure core modules are importable + # note: I needed to add "--ignore-installed certifi" because the + # installation started failing suddenly + pip install . --ignore-installed certifi + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests/cli/test_cloud_execution.py + + - run: | + echo "Integration tests... success! ;-)" + + # Update check run called "integration-fork" + - uses: actions/github-script@v5 + id: update-check-run + if: ${{ always() }} + env: + number: ${{ github.event.client_payload.pull_request.number }} + job: ${{ github.job }} + # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run + conclusion: ${{ job.status }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pull } = await github.rest.pulls.get({ + ...context.repo, + pull_number: process.env.number + }); + const ref = pull.head.sha; + + const { data: checks } = await github.rest.checks.listForRef({ + ...context.repo, + ref + }); + + const check = checks.check_runs.filter(c => c.name === process.env.job); + + const { data: result } = await github.rest.checks.update({ + ...context.repo, + check_run_id: check[0].id, + status: 'completed', + conclusion: process.env.conclusion + }); + + return result; + + + integration-trusted-windows: + runs-on: windows-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + steps: + + - name: Start PostgreSQL + run: | + $pgService = Get-Service -Name postgresql* + Set-Service -InputObject $pgService -Status running -StartupType automatic + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + - name: Change PostgreSQL password + run: | + & $env:PGBIN\psql --command "ALTER USER postgres PASSWORD 'postgres'" + - name: Check PostgreSQL authentication + run: | + $env:PGPASSWORD = 'postgres' + & $env:PGBIN\pg_isready --dbname postgres --username postgres + + - name: Branch based PR checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash -l {0} + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + # there's a bug in jupyter, we need to install this first + conda install pywin32 + + # install graphviz (pygraphviz not available in conda for win-64, + # it will be installed from pip) + # conda install -y graphviz -c conda-forge + # note: this installation still gives trouble (error when saving png + # files but it makes pygraphviz importable), we can get rid of this + # once we update the tests to mock the @requires decorator to make + # it think that pygraphviz is installed when it's not. the tests + # currently do not call pygraphviz (it's mocked), but @requires imports + conda install -y -c alubbock pygraphviz + # seems like pygraphviz just doesnt work on windows + # https://github.com/pygraphviz/pygraphviz/issues/40 + # pip install pygraphviz + + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # https://github.com/ipython/ipython/issues/12049 + pip install prompt-toolkit==2.0.10 + + # install R and IR kernel + conda install -y r-base r-irkernel -c conda-forge + echo "IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + shell: bash -l {0} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests/cli/test_cloud_execution.py + + # Repo owner has commented /ok-to-test on a (fork-based) pull request + integration-fork-windows: + runs-on: windows-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + if: + github.event_name == 'repository_dispatch' && + github.event.client_payload.slash_command.sha != '' && + contains(github.event.client_payload.pull_request.head.sha, github.event.client_payload.slash_command.sha) + steps: + + # Check out merge commit + - name: Fork based /ok-to-test checkout + uses: actions/checkout@v2 + with: + ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' + + - name: Start PostgreSQL + run: | + $pgService = Get-Service -Name postgresql* + Set-Service -InputObject $pgService -Status running -StartupType automatic + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + - name: Change PostgreSQL password + run: | + & $env:PGBIN\psql --command "ALTER USER postgres PASSWORD 'postgres'" + - name: Check PostgreSQL authentication + run: | + $env:PGPASSWORD = 'postgres' + & $env:PGBIN\pg_isready --dbname postgres --username postgres + + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash -l {0} + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + # there's a bug in jupyter, we need to install this first + conda install pywin32 + + # install graphviz (pygraphviz not available in conda for win-64, + # it will be installed from pip) + # conda install -y graphviz -c conda-forge + # note: this installation still gives trouble (error when saving png + # files but it makes pygraphviz importable), we can get rid of this + # once we update the tests to mock the @requires decorator to make + # it think that pygraphviz is installed when it's not. the tests + # currently do not call pygraphviz (it's mocked), but @requires imports + conda install -y -c alubbock pygraphviz + # seems like pygraphviz just doesnt work on windows + # https://github.com/pygraphviz/pygraphviz/issues/40 + # pip install pygraphviz + + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # https://github.com/ipython/ipython/issues/12049 + pip install prompt-toolkit==2.0.10 + + # install R and IR kernel + conda install -y r-base r-irkernel -c conda-forge + echo "IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + shell: bash -l {0} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests/cli/test_cloud_execution.py + + - run: | + echo "Integration tests... success! ;-)" + + # Update check run called "integration-fork" + - uses: actions/github-script@v5 + id: update-check-run + if: ${{ always() }} + env: + number: ${{ github.event.client_payload.pull_request.number }} + job: ${{ github.job }} + # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run + conclusion: ${{ job.status }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { data: pull } = await github.rest.pulls.get({ + ...context.repo, + pull_number: process.env.number + }); + const ref = pull.head.sha; + + const { data: checks } = await github.rest.checks.listForRef({ + ...context.repo, + ref + }); + + const check = checks.check_runs.filter(c => c.name === process.env.job); + + const { data: result } = await github.rest.checks.update({ + ...context.repo, + check_run_id: check[0].id, + status: 'completed', + conclusion: process.env.conclusion + }); + + return result; \ No newline at end of file diff --git a/.github/workflows/ci-unit.yml b/.github/workflows/ci-unit.yml new file mode 100644 index 000000000..57ff0a821 --- /dev/null +++ b/.github/workflows/ci-unit.yml @@ -0,0 +1,254 @@ +# Run unit tests on any branch/fork pull request +on: + pull_request + +name: Unit tests + +jobs: + unit-linux: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Config + run: | + conda create -q -n conda-env python=${{ matrix.python-version }} + # for the tests that depend on git repos + git config --global user.email "someone@example.com" + git config --global user.name "Someone" + - name: Install dependencies + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + # for some reason, conda does not resolve the env correctly and + # installs and old version of pygraphviz (1.3) which breaks in + # Python 3.7 + conda install -y "pygraphviz>=1.6" -c conda-forge + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + python -c 'import ploomber.io' + # install dev requirements + pip install .[dev] + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + # install sample package for tests + pip install --editable tests/assets/test_pkg + - name: Run tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + PLOOMBER_STATS_ENABLED: false + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + # doc tests + export PY_IGNORE_IMPORTMISMATCH=1 + pytest src/ploomber --doctest-modules --ignore src/ploomber/resources + unset PY_IGNORE_IMPORTMISMATCH + # unit tests + pytest tests --cov=ploomber --ignore=tests/cli/test_cloud_execution.py + coveralls + + + - run: echo "Unit tests... success on linux!" + + unit-macos: + runs-on: macos-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + + steps: + # https://www.cybertec-postgresql.com/en/postgresql-github-actions-continuous-integration/ + - name: Start PostgreSQL + run: | + brew services start postgresql + echo "Check PostgreSQL service is running" + i=10 + COMMAND='pg_isready' + while [ $i -gt 0 ]; do + echo "Check PostgreSQL service status" + eval $COMMAND && break + ((i--)) + if [ $i == 0 ]; then + echo "PostgreSQL service not ready, all attempts exhausted" + exit 1 + fi + echo "PostgreSQL service not ready, wait 10 more sec, attempts left: $i" + sleep 10 + done + + - name: Create PostgreSQL user + run: psql --command="CREATE USER postgres PASSWORD 'postgres'" postgres + + - name: Update PostgreSQL permissions + run: psql --command="ALTER DATABASE postgres OWNER TO postgres" postgres + + - name: Check PostgreSQL authentication + run: PGPASSWORD=postgres pg_isready --dbname postgres --username postgres + + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + + - name: Set up R + uses: r-lib/actions/setup-r@v2 + + - name: Install dependencies + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + conda install -y pygraphviz -c conda-forge + + # Test vanilla installation, make sure core modules are importable + # note: I needed to add "--ignore-installed certifi" because the + # installation started failing suddenly + pip install . --ignore-installed certifi + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # install IR kernel + echo "install.packages('IRkernel', repos='http://cran.us.r-project.org'); IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests --ignore=tests/cli/test_cloud_execution.py + + unit-windows: + runs-on: windows-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + + steps: + # https://www.cybertec-postgresql.com/en/postgresql-github-actions-continuous-integration/ + - name: Start PostgreSQL + run: | + $pgService = Get-Service -Name postgresql* + Set-Service -InputObject $pgService -Status running -StartupType automatic + Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru + - name: Change PostgreSQL password + run: | + & $env:PGBIN\psql --command "ALTER USER postgres PASSWORD 'postgres'" + - name: Check PostgreSQL authentication + run: | + $env:PGPASSWORD = 'postgres' + & $env:PGBIN\pg_isready --dbname postgres --username postgres + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + activate-environment: conda-env + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash -l {0} + run: | + + eval "$(conda shell.bash hook)" + conda activate conda-env + + # make the build fail quickly on flake8 errors + pip install flake8 + flake8 + + # there's a bug in jupyter, we need to install this first + conda install pywin32 + + # install graphviz (pygraphviz not available in conda for win-64, + # it will be installed from pip) + # conda install -y graphviz -c conda-forge + # note: this installation still gives trouble (error when saving png + # files but it makes pygraphviz importable), we can get rid of this + # once we update the tests to mock the @requires decorator to make + # it think that pygraphviz is installed when it's not. the tests + # currently do not call pygraphviz (it's mocked), but @requires imports + conda install -y -c alubbock pygraphviz + # seems like pygraphviz just doesnt work on windows + # https://github.com/pygraphviz/pygraphviz/issues/40 + # pip install pygraphviz + + # Test vanilla installation, make sure core modules are importable + pip install . + python -c 'import ploomber' + python -c 'import ploomber.tasks' + python -c 'import ploomber.products' + python -c 'import ploomber.clients' + + # install dev requirements + pip install .[dev] + + # https://github.com/ipython/ipython/issues/12049 + pip install prompt-toolkit==2.0.10 + + # install R and IR kernel + conda install -y r-base r-irkernel -c conda-forge + echo "IRkernel::installspec()" | Rscript - + + # install sample package for tests + pip install --editable tests/assets/test_pkg + + - name: Run tests + env: + PLOOMBER_STATS_ENABLED: false + PLOOMBER_CLOUD_KEY: ${{ secrets.PLOOMBER_CLOUD_KEY }} + shell: bash -l {0} + run: | + eval "$(conda shell.bash hook)" + conda activate conda-env + pytest tests --ignore=tests/cli/test_cloud_execution.py \ No newline at end of file diff --git a/.github/workflows/ok-to-test.yml b/.github/workflows/ok-to-test.yml new file mode 100644 index 000000000..4f026ba40 --- /dev/null +++ b/.github/workflows/ok-to-test.yml @@ -0,0 +1,42 @@ +# If someone with write access comments "/ok-to-test" on a pull request, emit a repository_dispatch event +name: Ok To Test + +on: + issue_comment: + types: [created] + +jobs: + print-jobs: + runs-on: ubuntu-latest + steps: + - run: | + echo "THIS IS tests... success! ;-)" + + ok-to-test: + runs-on: ubuntu-latest + # Only run for PRs, not issue comments + # if: ${{ github.event.issue.pull_request }} + steps: + # Generate a GitHub App installation access token from an App ID and private key + # To create a new GitHub App: + # https://developer.github.com/apps/building-github-apps/creating-a-github-app/ + # See app.yml for an example app manifest + # - name: Generate token + # id: generate_token + # uses: tibdex/github-app-token@v1 + # with: + # app_id: ${{ secrets.APP_ID }} + # private_key: ${{ secrets.PRIVATE_KEY }} + + - name: Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v1 + env: + TOKEN: ${{ steps.generate_token.outputs.token }} + with: + #token: ${{ env.TOKEN }} # GitHub App installation access token + token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} # PAT or OAuth token will also work + reaction-token: ${{ secrets.GITHUB_TOKEN }} + issue-type: pull-request + commands: ok-to-test + named-args: true + permission: write