From 777a8453ccd103070565691dc998d91ab76e1e6e Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 2 Sep 2021 17:30:24 -0400 Subject: [PATCH 01/31] Start branch for 0.15.9 --- CHANGELOG.md | 5 ++++- xdoctest/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf12a9e3..4431cff2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ We are currently working on porting this changelog to the specifications in [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Version 0.15.8 - Unreleased +## Version 0.15.9 - Unreleased + + +## Version 0.15.8 - Released 2021-09-02 ### Fixed diff --git a/xdoctest/__init__.py b/xdoctest/__init__.py index 0246ef23..e8aa68aa 100644 --- a/xdoctest/__init__.py +++ b/xdoctest/__init__.py @@ -280,7 +280,7 @@ def fib(n): mkinit xdoctest --nomods ''' -__version__ = '0.15.8' +__version__ = '0.15.9' # Expose only select submodules From 09aa197c1cea06bfb31fa952bf53c471e2449a9a Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 2 Sep 2021 17:32:52 -0400 Subject: [PATCH 02/31] wip --- CHANGELOG.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4431cff2..dc93b53f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,6 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.15.8 - Released 2021-09-02 -### Fixed - -* Hotfix - removed debug print statements - - -## Version 0.15.7 - Released 2021-09-02 - ### Changed * Removed the distracting and very long internal traceback that occurred in pytest when a module errors while it is being imported before the doctest is @@ -25,6 +18,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm nothing unless `-s` is also given so pytest does not supress output) +## Version 0.15.7 - Yanked + + ### Fixed * Bug in REQUIRES state did not respect `python_implementation` arguments * Ported sphinx fixes from ubelt From 0034be752cd7f85363dfda5a324c81093ad203af Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 2 Sep 2021 18:39:31 -0400 Subject: [PATCH 03/31] Cleanup changelog --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc93b53f..e49f758c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,6 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.15.8 - Released 2021-09-02 - ### Changed * Removed the distracting and very long internal traceback that occurred in pytest when a module errors while it is being imported before the doctest is @@ -20,14 +19,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.15.7 - Yanked - ### Fixed * Bug in REQUIRES state did not respect `python_implementation` arguments * Ported sphinx fixes from ubelt ## Version 0.15.6 - Released 2021-08-08 - ### Changed * Directive syntax errors are now handled as doctest runtime errors and return better debugging information. From ec14218904fdd1c06597277deb11c8d3bb33c675 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 3 Sep 2021 07:27:22 +0200 Subject: [PATCH 04/31] Fix test failure if pytest's flaky plugin is installed Disable pytest's flaky plugin in test_simple_pytest_import_error_cli in order to fix a test failure due to it mangling pytest's return code on import error. --- testing/test_pytest_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/test_pytest_cli.py b/testing/test_pytest_cli.py index 25c83636..8ef92510 100644 --- a/testing/test_pytest_cli.py +++ b/testing/test_pytest_cli.py @@ -67,7 +67,7 @@ def module_func1(): """ ''') temp_module = util_misc.TempModule(module_text, modname='imperr_test_mod') - command = sys.executable + ' -m pytest -v -s --xdoctest-verbose=3 --xdoctest ' + temp_module.dpath + command = sys.executable + ' -m pytest -p no:flaky -v -s --xdoctest-verbose=3 --xdoctest ' + temp_module.dpath print(command) info = cmd(command) print(info['out']) From 93ae000fcca5026be5ceaedc526ff9fc09967649 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 12:35:58 -0400 Subject: [PATCH 05/31] Cleanup pytest_cli tests add more asserts --- testing/test_pytest_cli.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/testing/test_pytest_cli.py b/testing/test_pytest_cli.py index 8ef92510..c187d7aa 100644 --- a/testing/test_pytest_cli.py +++ b/testing/test_pytest_cli.py @@ -45,11 +45,8 @@ def test_simple_pytest_import_error_cli(): This test case triggers an excessively long callback in xdoctest < dev/0.15.7 - xdoctest ~/code/xdoctest/testing/test_pytest_cli.py test_simple_pytest_import_error_cli - - import sys, ubelt - sys.path.append(ubelt.expandpath('~/code/xdoctest/testing')) - from test_pytest_cli import * # NOQA + CommandLine: + xdoctest ~/code/xdoctest/testing/test_pytest_cli.py test_simple_pytest_import_error_cli """ module_text = utils.codeblock( ''' @@ -67,15 +64,14 @@ def module_func1(): """ ''') temp_module = util_misc.TempModule(module_text, modname='imperr_test_mod') - command = sys.executable + ' -m pytest -p no:flaky -v -s --xdoctest-verbose=3 --xdoctest ' + temp_module.dpath + command = sys.executable + ' -m pytest -v -s --xdoctest-verbose=3 --xdoctest ' + temp_module.dpath print(command) info = cmd(command) + # We patched doctest_example so it no longer outputs this in the traceback + assert 'util_import' not in info['out'] print(info['out']) - - # info = cmd('pytest --xdoctest ' + temp_module.modpath) - # print(info['out']) - - assert info['ret'] == 1 + # Note: flaky changes the return code from 1 to 3, so test non-zero + assert info['ret'] != 0 def test_simple_pytest_syntax_error_cli(): @@ -96,9 +92,11 @@ def module_func1(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest --xdoctest ' + temp_module.dpath) print(info['out']) + assert info['ret'] != 0 info = cmd(sys.executable + ' -m pytest --xdoctest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 def test_simple_pytest_import_error_no_xdoctest(): @@ -114,10 +112,11 @@ def test_this(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 - info = cmd('pytest ' + temp_module.dpath) + info = cmd(sys.executable + ' -m pytest ' + temp_module.dpath) print(info['out']) - # assert info['ret'] == 0 + assert info['ret'] != 0 def test_simple_pytest_syntax_error_no_xdoctest(): @@ -133,7 +132,8 @@ def test_this(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 info = cmd(sys.executable + ' -m pytest ' + temp_module.dpath) print(info['out']) - # assert info['ret'] == 0 + assert info['ret'] != 0 From d4884ff3568bceaeb8bcd346e50048d62e2db043 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 13:39:45 -0400 Subject: [PATCH 06/31] Initial work on github actions wip wip wip wip --- .github/dependabot.yml | 8 + .github/workflows/setup_action_secrets.md | 217 ++++++++++++++++ .github/workflows/tests.yml | 292 ++++++++++++++++++++++ 3 files changed, 517 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/setup_action_secrets.md create mode 100644 .github/workflows/tests.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..6a61862a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "friday" diff --git a/.github/workflows/setup_action_secrets.md b/.github/workflows/setup_action_secrets.md new file mode 100644 index 00000000..6ee032a2 --- /dev/null +++ b/.github/workflows/setup_action_secrets.md @@ -0,0 +1,217 @@ +========================== +GITHUB ACTION INSTRUCTIONS +========================== + +This file is a reference script for setting up secrets for github actions +(because currently we can't add heredocs to github yaml files like we can with +other CI tools) + +This file was designed to be used as a template. You can adapt it to +new projects with a few simple changes. Namely perform the following +search and replaces. + + +# TODO: re-setup template + + +```bash +cat .github/workflows/setup_action_secrets.md | \ + sed 's|GITHUB_USER|Erotemic|g' | \ + sed 's|PYPKG|xdoctest|g' | \ + sed 's|GPG_ID|travis-ci-Erotemic|g' | \ + sed 's|PKG_CI_SECRET|EROTEMIC_CI_SECRET|g' \ +> /tmp/repl + +# Check the diff +colordiff .github/workflows/setup_action_secrets.md /tmp/repl + +# overwrite if you like the diff +cp /tmp/repl .github/workflows/setup_action_secrets.md +``` + +To use this script you need the following configurations on your CI account. + +NOTES +----- + +* This script will require maintenance for new releases of Python + + +CI SECRETS +---------- + +Almost all of the stages in this pipeline can be performed on a local machine +(making it much easier to debug) as well as the CI machine. However, there are +a handful of required environment variables which will contain sensitive +information. These variables are + +* `TWINE_USERNAME` - this is your pypi username + twine info is only needed if you want to automatically publish to pypi + +* `TWINE_PASSWORD` - this is your pypi password + +* `EROTEMIC_CI_SECRET` - We will use this as a secret key to encrypt/decrypt gpg secrets + This is only needed if you want to automatically sign published + wheels with a gpg key. + +* `PERSONAL_GITHUB_PUSH_TOKEN` - + This is only needed if you want to automatically git-tag release branches. + + To make a API token go to: + https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token + +Instructions: + + Browse to: + https://github.com/Erotemic/xdoctest/settings/secrets/actions + + Do whatever you need to locally access the values of these variables + + echo $TWINE_USERNAME + echo $PERSONAL_GITHUB_PUSH_TOKEN + echo $EROTEMIC_CI_SECRET + echo $TWINE_PASSWORD + + For each one, click "Add Environment Variable" and enter the name + and value. Unfortunately this is a manual process. + +WARNING: + +Ensure that your project settings do not allow Forks to view environment +variables. + +TODO: Can you protect branches on GithubActions? Is that the default? + +TODO: Look into secrethub + +WARNING: If an untrusted actor gains the ability to write to a +protected branch, then they will be able to exfiltrate your secrets. + +WARNING: These variables contain secret information. Ensure that these +the protected and masked settings are enabled when you create them. + + +ENCRYPTING GPG SECRETS +---------------------- + +The following script demonstrates how to securely encrypt a secret GPG key. It +is assumed that you have a file `secret_loader.sh` that looks like this + +```bash + source secretfile +``` + +and then a secret file that looks like this + +```bash + #!/bin/bash + echo /some/secret/file + + export TWINE_USERNAME= + export TWINE_PASSWORD= + export EROTEMIC_CI_SECRET="" + export PERSONAL_GITHUB_PUSH_TOKEN="git-push-token:" +``` + +You should also make a `secret_unloader.sh` that points to a script that +unloads these secret variables from the environment. + +Given this file-structure setup, you can then run the following +commands verbatim. Alternatively just populate the environment +variables and run line-by-line without creating the secret +loader/unloader scripts. + +```bash +# THIS IS NOT EXECUTE ON THE CI, THIS IS FOR DEVELOPER REFERENCE +# ON HOW THE ENCRYPTED GPG KEYS ARE SETUP. + +# Load or generate secrets +source $(secret_loader.sh) +echo $EROTEMIC_CI_SECRET +echo $TWINE_USERNAME + +# ADD RELEVANT VARIABLES TO CIRCLECI SECRET VARIABLES +# https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables +# See previous CIRCLE_CI section for more details + +# HOW TO ENCRYPT YOUR SECRET GPG KEY +IDENTIFIER="travis-ci-Erotemic" +GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}' | tail -c 9) +echo "GPG_KEYID = $GPG_KEYID" + +# Export plaintext gpg public keys, private keys, and trust info +mkdir -p dev +gpg --armor --export-secret-keys $GPG_KEYID > dev/ci_secret_gpg_key.pgp +gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp +gpg --export-ownertrust > dev/gpg_owner_trust + +# Encrypt gpg keys and trust with CI secret +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_key.pgp > dev/ci_secret_gpg_key.pgp.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc +echo $GPG_KEYID > dev/public_gpg_key + +# Test decrpyt +cat dev/public_gpg_key +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc + +source $(secret_unloader.sh) + +# Look at what we did, clean up, and add it to git +ls dev/*.enc +rm dev/gpg_owner_trust dev/*.pgp +git status +git add dev/*.enc +git add dev/public_gpg_key +``` + + +Test Github Push Token +---------------------- + +The following script tests if your `PERSONAL_GITHUB_PUSH_TOKEN` environment variable is correctly setup. + +```bash +docker run -it ubuntu +apt update -y && apt install git -y +git clone https://github.com/Erotemic/xdoctest.git +cd xdoctest +# do sed twice to handle the case of https clone with and without a read token +git config user.email "ci@circleci.com" +git config user.name "CircleCI-User" +URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g') +echo "URL_HOST = $URL_HOST" +git tag "test-tag4" +git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" + +# Cleanup after you verify the tags shows up on the remote +git push --delete origin test-tag4 +git tag --delete test-tag4 +``` + + + +Github Action Local Test +------------------------ + + +```bash + # How to run locally + # https://packaging.python.org/guides/using-testpypi/ + cd $HOME/code + git clone https://github.com/nektos/act.git $HOME/code/act + cd $HOME/code/act + chmod +x install.sh + ./install.sh -b $HOME/.local/opt/act + cd $HOME/code/xdoctest + + load_secrets + unset GITHUB_TOKEN + $HOME/.local/opt/act/act \ + --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \ + --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \ + --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \ + --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \ + --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..e503dd8e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,292 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Tests + +on: + push: + pull_request: + branches: [ master ] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./xdoctest --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=20 --max-line-length=127 --statistics + + build_and_test_sdist: + name: Test sdist Python 3.8 + runs-on: ubuntu-latest + needs: [lint] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Upgrade pip + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements/tests.txt + python -m pip install -r requirements/runtime.txt + - name: Build sdist + run: | + python setup.py sdist + - name: Install sdist + run: | + cd dist + ls -al + pip install xdoctest*.tar.gz -v + # Ensure the source doesn't conflict with the test + rm -rf xdoctest + - name: Test minimal sdist + run: | + pwd + ls -al + python run_tests.py + - name: Test full sdist + run: | + pwd + ls -al + python -m pip install -r requirements/optional.txt + python run_tests.py + + - name: Upload sdist artifact + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./dist/*.tar.gz + + build_and_test_wheels: + name: ${{ matrix.cibw_build }} on ${{ matrix.os }}, arch=${{ matrix.arch }} + runs-on: ${{ matrix.os }} + needs: [lint] + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + arch: [auto] + cibw_build: [cp3*-*] + cibw_skip: ["*-win32"] + # Add additional workers to reduce overall build time + include: + - os: windows-latest + cibw_build: cp3*-win32 + arch: auto + cibw_skip: "" + - os: ubuntu-latest + arch: aarch64 + cibw_build: cp35-* + - os: ubuntu-latest + arch: aarch64 + cibw_build: cp36-* + - os: ubuntu-latest + arch: aarch64 + cibw_build: cp37-* + - os: ubuntu-latest + arch: aarch64 + cibw_build: cp38-* + - os: ubuntu-latest + arch: aarch64 + cibw_build: cp39-* + + + steps: + - name: Checkout source + uses: actions/checkout@v2 + + # Configure compilers for Windows 64bit. + - name: Enable MSVC 64bit + if: matrix.os == 'windows-latest' && matrix.cibw_build != 'cp3*-win32' + uses: ilammy/msvc-dev-cmd@v1 + + # Configure compilers for Windows 32bit. + - name: Enable MSVC 32bit + if: matrix.os == 'windows-latest' && matrix.cibw_build == 'cp3*-win32' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + # Emulate aarch64 ppc64le s390x under linux + - name: Set up QEMU + if: runner.os == 'Linux' && matrix.arch != 'auto' + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml + #- name: Build wheels + # uses: pypa/cibuildwheel@v1.11.0 + # with: + # output-dir: wheelhouse + # # to supply options, put them in 'env', like: + # env: + # CIBW_SKIP: ${{ matrix.cibw_skip }} + # CIBW_BUILD: ${{ matrix.cibw_build }} + # CIBW_TEST_REQUIRES: -r requirements/tests.txt + # CIBW_TEST_COMMAND: python {project}/run_tests.py + # # configure cibuildwheel to build native archs ('auto'), or emulated ones + # CIBW_ARCHS_LINUX: ${{ matrix.arch }} + + - name: Build pure wheel + shell: bash + run: | + python -m pip install setuptools>=0.8 wheel + python -m pip wheel --wheel-dir wheelhouse . + + - name: Test pure wheel + shell: bash + run: | + # Remove source directory + rm -rf xdoctest + # Install the wheel + python -m pip install wheelhouse/xdoctest*.whl + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + python -m pip install -r requirements/tests.txt + # Run the tests + python -m pytest -p pytester -p no:doctest --xdoctest --cov-config .coveragerc --cov-report html --cov-report term --cov=xdoctest $XDOCTEST_DPATH ./testing + + - name: Show built files + shell: bash + run: ls -la wheelhouse + + - name: Set up Python 3.8 to combine coverage Linux + if: runner.os == 'Linux' + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Combine coverage Linux + if: runner.os == 'Linux' + run: | + echo '############ PWD' + pwd + python -m pip install coverage[toml] + echo '############ combine' + coverage combine ./wheelhouse + echo '############ XML' + coverage xml -o ./tests/coverage.xml + echo '############ FIND' + find . -name .coverage.* + find . -name coverage.xml + + - name: Codecov Upload + uses: codecov/codecov-action@v1 + with: + file: ./tests/coverage.xml + + - name: Upload wheels artifact + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/xdoctest*.whl + + deploy: + name: Uploading to PyPi + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + needs: [build_and_test_wheels, build_and_test_sdist] + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Download wheels and sdist + uses: actions/download-artifact@v2 + with: + name: wheels + path: wheelhouse + + - name: Show files to upload + shell: bash + run: ls -la wheelhouse + - name: Sign and Publish + env: + # Secrets should be uploaded here: + # https://github.com/Erotemic/xdoctest/settings/secrets/actions + # Toggle comments to publish to the test pypi instead of the real one + #TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + #EROTEMIC_TWINE_USERNAME: ${{ secrets.EROTEMIC_TWINE_USERNAME }} + #EROTEMIC_TWINE_PASSWORD: ${{ secrets.EROTEMIC_TWINE_PASSWORD }} + TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ + EROTEMIC_TEST_TWINE_USERNAME: ${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} + EROTEMIC_TEST_TWINE_PASSWORD: ${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} + run: | + ls -al + GPG_EXECUTABLE=gpg + $GPG_EXECUTABLE --version + openssl version + $GPG_EXECUTABLE --list-keys + export EROTEMIC_CI_SECRET=${{ secrets.EROTEMIC_CI_SECRET }} + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" + $GPG_EXECUTABLE --list-keys + MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") + VERSION=$(python -c "import setup; print(setup.VERSION)") + pip install twine + pip install six pyopenssl ndg-httpsclient pyasn1 -U --user + pip install requests[security] twine --user + GPG_KEYID=$(cat dev/public_gpg_key) + echo "GPG_KEYID = '$GPG_KEYID'" + # + #export TWINE_REPOSITORY_URL=https://test.pypi.org/legacy/ + #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} + #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} + #MB_PYTHON_TAG=$MB_PYTHON_TAG \ + # DO_GPG=True GPG_KEYID=$GPG_KEYID \ + # TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ + # TWINE_USERNAME=$TEST_TWINE_USERNAME \ + # TWINE_PASSWORD=$TEST_TWINE_PASSWORD \ + # GPG_EXECUTABLE=$GPG_EXECUTABLE \ + # DO_UPLOAD=True \ + # DO_TAG=False ./publish.sh + # + export TWINE_REPOSITORY_URL=https://upload.pypi.org/legacy/ + export TWINE_USERNAME=${{ secrets.EROTEMIC_TWINE_USERNAME }} + export TWINE_PASSWORD=${{ secrets.EROTEMIC_TWINE_PASSWORD }} + MB_PYTHON_TAG="*" \ + DO_GPG=True GPG_KEYID=$GPG_KEYID \ + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ + TWINE_USERNAME=$TWINE_USERNAME \ + TWINE_PASSWORD=$TWINE_PASSWORD \ + GPG_EXECUTABLE=$GPG_EXECUTABLE \ + DO_UPLOAD=False \ + DO_TAG=False ./publish.sh + +### +# Unfortunately we cant (yet) use the yaml docstring trick here +# https://github.community/t/allow-unused-keys-in-workflow-yaml-files/172120 +#__doc__: | +# # How to run locally +# # https://packaging.python.org/guides/using-testpypi/ +# cd $HOME/code +# git clone https://github.com/nektos/act.git $HOME/code/act +# cd $HOME/code/act +# chmod +x install.sh +# ./install.sh -b $HOME/.local/opt/act +# cd $HOME/code/xdoctest + +# load_secrets +# unset GITHUB_TOKEN +# $HOME/.local/opt/act/act \ +# --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \ +# --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \ +# --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \ +# --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \ +# --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD From ae945701003c53f3b607bf243682efe001092671 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 14:43:58 -0400 Subject: [PATCH 07/31] Fix plugin test wip wip wip wip wip wip wip wip wip wip wip wip wip wip wip --- .github/workflows/tests.yml | 110 +++++++++++++++++++++++------------- pytest.ini | 2 +- testing/test_plugin.py | 9 +-- 3 files changed, 78 insertions(+), 43 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e503dd8e..17f69e38 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,7 @@ on: branches: [ master ] jobs: + lint: runs-on: ubuntu-latest @@ -32,7 +33,7 @@ jobs: build_and_test_sdist: name: Test sdist Python 3.8 runs-on: ubuntu-latest - needs: [lint] + #needs: [lint] steps: - uses: actions/checkout@v2 - name: Set up Python 3.8 @@ -49,22 +50,30 @@ jobs: python setup.py sdist - name: Install sdist run: | - cd dist - ls -al - pip install xdoctest*.tar.gz -v - # Ensure the source doesn't conflict with the test - rm -rf xdoctest + ls -al ./dist + pip install dist/xdoctest*.tar.gz -v + mv xdoctest ignore_src_xdoctest - name: Test minimal sdist run: | pwd ls -al - python run_tests.py + # Run the tests + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest --cov=xdoctest $XDOCTEST_DPATH ./testing + - name: Test full sdist run: | pwd ls -al python -m pip install -r requirements/optional.txt - python run_tests.py + # Run the tests + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest $XDOCTEST_DPATH ./testing + - name: Upload sdist artifact uses: actions/upload-artifact@v2 @@ -73,36 +82,42 @@ jobs: path: ./dist/*.tar.gz build_and_test_wheels: - name: ${{ matrix.cibw_build }} on ${{ matrix.os }}, arch=${{ matrix.arch }} + # TODO: handle different python versions: https://github.com/actions/setup-python + name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }} runs-on: ${{ matrix.os }} - needs: [lint] + #needs: [lint] strategy: matrix: os: [ubuntu-latest, windows-latest, macOS-latest] + python-version: + - '3.5' + - '3.6' + - '3.7' + - '3.8' arch: [auto] - cibw_build: [cp3*-*] - cibw_skip: ["*-win32"] + #cibw_build: [cp3*-*] + #cibw_skip: ["*-win32"] # Add additional workers to reduce overall build time - include: - - os: windows-latest - cibw_build: cp3*-win32 - arch: auto - cibw_skip: "" - - os: ubuntu-latest - arch: aarch64 - cibw_build: cp35-* - - os: ubuntu-latest - arch: aarch64 - cibw_build: cp36-* - - os: ubuntu-latest - arch: aarch64 - cibw_build: cp37-* - - os: ubuntu-latest - arch: aarch64 - cibw_build: cp38-* - - os: ubuntu-latest - arch: aarch64 - cibw_build: cp39-* + #include: + # - os: windows-latest + # cibw_build: cp3*-win32 + # arch: auto + # cibw_skip: "" + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp35-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp36-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp37-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp38-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp39-* steps: @@ -128,6 +143,7 @@ jobs: with: platforms: all + # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml #- name: Build wheels # uses: pypa/cibuildwheel@v1.11.0 @@ -142,6 +158,11 @@ jobs: # # configure cibuildwheel to build native archs ('auto'), or emulated ones # CIBW_ARCHS_LINUX: ${{ matrix.arch }} + # Setup Python environemtn for a pure wheel + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Build pure wheel shell: bash run: | @@ -150,16 +171,28 @@ jobs: - name: Test pure wheel shell: bash + env: + CI_PYTHON_VERSION: py${{ matrix.python-version }} + #OS_NAME: ${{ matrix.os }} run: | - # Remove source directory - rm -rf xdoctest + # Remove source directory (ensure it doesn't conflict) + mv xdoctest ignore_src_xdoctest + rm pytest.ini # Install the wheel python -m pip install wheelhouse/xdoctest*.whl - # Get path to installed package - XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") python -m pip install -r requirements/tests.txt + # Run in a sandboxed directory + WORKSPACE_DNAME="testdir_${CI_PYTHON_VERSION}_${GITHUB_RUN_ID}_${RUNNER_OS}" + mkdir -p $WORKSPACE_DNAME + cd $WORKSPACE_DNAME # Run the tests - python -m pytest -p pytester -p no:doctest --xdoctest --cov-config .coveragerc --cov-report html --cov-report term --cov=xdoctest $XDOCTEST_DPATH ./testing + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest --cov-config ../.coveragerc --cov-report term --cov=xdoctest $XDOCTEST_DPATH ../testing + mv .coverage "../.coverage.$WORKSPACE_DNAME" + cd .. + # Move coverage file to a new name - name: Show built files shell: bash @@ -176,9 +209,10 @@ jobs: run: | echo '############ PWD' pwd + ls -al python -m pip install coverage[toml] echo '############ combine' - coverage combine ./wheelhouse + coverage combine . echo '############ XML' coverage xml -o ./tests/coverage.xml echo '############ FIND' diff --git a/pytest.ini b/pytest.ini index ed5ece45..b25ad93c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ # ON COVERAGE OF PYTEST PLUGINS: # http://pytest-cov.readthedocs.io/en/latest/plugins.html addopts = -p pytester -p no:doctest --xdoctest --ignore-glob=setup.py --ignore=.tox --ignore=setup.py --ignore=dev -norecursedirs = .git ignore build __pycache__ docs *.egg-info _* dev testing/pybind11_test setup.py +norecursedirs = .git ignore build __pycache__ docs *.egg-info dev testing/pybind11_test setup.py # --pyargs --doctest-modules --ignore=.tox ;rsyncdirs = tox.ini pytest.py _pytest testing ;python_files = test_*.py *_test.py testing/*/*.py diff --git a/testing/test_plugin.py b/testing/test_plugin.py index a08e022d..e6421422 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -235,12 +235,13 @@ def test_encoding(self, testdir, test_string, encoding): [pytest] xdoctest_encoding={0} """.format(encoding)) - xdoctest = u""" + doctest = u""" >>> u"{0}" {1} """.format(test_string, repr(test_string)) - testdir._makefile(".txt", [xdoctest], {}, encoding=encoding) + print(doctest) + testdir._makefile(".txt", [doctest], {}, encoding=encoding) result = testdir.runpytest(*(EXTRA_ARGS + OLD_TEXT_ARGS)) @@ -257,7 +258,7 @@ def test_encoding(self, pytester, test_string, encoding): pytester.makeini( """ [pytest] - doctest_encoding={} + xdoctest_encoding={} """.format( encoding ) @@ -271,7 +272,7 @@ def test_encoding(self, pytester, test_string, encoding): fn = pytester.path / "test_encoding.txt" fn.write_text(doctest, encoding=encoding) - result = pytester.runpytest() + result = pytester.runpytest(*OLD_TEXT_ARGS) result.stdout.fnmatch_lines(["*1 passed*"]) From 98fdced82e5752f869520b1236a60d642ca46d5d Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 17:27:12 -0400 Subject: [PATCH 08/31] Fix pytest plugin tests --- run_tests.py | 3 ++- testing/test_plugin.py | 33 +++++++++++++++++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/run_tests.py b/run_tests.py index f413ee8e..06df7e5c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,6 +4,7 @@ import pytest import sys package_name = 'xdoctest' + package_dpath = package_name pytest_args = [ '-p', 'pytester', '-p', 'no:doctest', @@ -12,7 +13,7 @@ '--cov-report', 'html', '--cov-report', 'term', '--cov=' + package_name, - package_name, 'testing' + package_dpath, 'testing' ] pytest_args = pytest_args + sys.argv[1:] print('pytest.__version__ = {!r}'.format(pytest.__version__)) diff --git a/testing/test_plugin.py b/testing/test_plugin.py index e6421422..33aa2a21 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -2,6 +2,7 @@ """ Adapted from the original `pytest/testing/test_doctest.py` module at: https://github.com/pytest-dev/pytest + https://github.com/pytest-dev/pytest/blob/main/testing/test_doctest.py """ from __future__ import print_function, division, absolute_import, unicode_literals import sys @@ -229,7 +230,7 @@ def test_encoding(self, testdir, test_string, encoding): """Test support for xdoctest_encoding ini option. CommandLine: - pytest testing/test_plugin.py::TestXDoctest::test_encoding + pytest testing/test_plugin.py::TestXDoctest::test_encoding -s -v """ testdir.makeini(""" [pytest] @@ -243,7 +244,7 @@ def test_encoding(self, testdir, test_string, encoding): print(doctest) testdir._makefile(".txt", [doctest], {}, encoding=encoding) - result = testdir.runpytest(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '*1 passed*', @@ -272,7 +273,7 @@ def test_encoding(self, pytester, test_string, encoding): fn = pytester.path / "test_encoding.txt" fn.write_text(doctest, encoding=encoding) - result = pytester.runpytest(*OLD_TEXT_ARGS) + result = pytester.runpytest("--xdoctest", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(["*1 passed*"]) @@ -536,7 +537,7 @@ def test_doctest_unex_importerror_only_txt(self, testdir): testdir.maketxtfile(""" >>> import asdalsdkjaslkdjasd """) - result = testdir.runpytest(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) # xdoctest is never executed because of error during hello.py collection result.stdout.fnmatch_lines([ "*>>> import asdals*", @@ -628,14 +629,14 @@ def somefunc(): def test_txtfile_failing(self, testdir): """ CommandLine: - pytest testing/test_plugin.py::TestXDoctest::test_txtfile_failing + pytest testing/test_plugin.py::TestXDoctest::test_txtfile_failing -s """ p = testdir.maketxtfile(""" >>> i = 0 >>> i + 1 2 """) - result = testdir.runpytest(p, "-s", *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", "-s", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '*1 >>> i = 0', '*2 >>> i + 1', @@ -658,7 +659,7 @@ def test_txtfile_with_fixtures(self, testdir): >>> type(dir).__name__ 'LocalPath' """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) def test_txtfile_with_usefixtures_in_ini(self, testdir): @@ -682,7 +683,7 @@ def myfixture(monkeypatch): >>> os.environ["HELLO"] 'WORLD' """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) def test_ignored_whitespace(self, testdir): @@ -784,7 +785,7 @@ def test_xdoctest_multiline_list(self, testdir): >>> print(len(x)) 6 """) - result = testdir.runpytest(p, *EXTRA_ARGS) + result = testdir.runpytest(p, "--xdoctest-modules", *EXTRA_ARGS) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_multiline_string(self, testdir): @@ -818,7 +819,7 @@ def test_xdoctest_multiline_string(self, testdir): >>> '''.strip()) Just prefix everything with >>> and the xdoctest should work """).lstrip()) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_trycatch(self, testdir): @@ -849,7 +850,7 @@ def test_xdoctest_trycatch(self, testdir): foo bar """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_functions(self, testdir): @@ -872,7 +873,7 @@ def test_xdoctest_functions(self, testdir): >>> func() now the ast parser makes doctests nice for us """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_stdout_capture_no(self, testdir): @@ -1093,7 +1094,7 @@ def test_unicode_string(self, testdir): >>> b'12'.decode('ascii') '12' """) - reprec = testdir.inline_run(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) passed = int(sys.version_info[0] >= 3) reprec.assertoutcome(passed=passed, failed=int(not passed)) @@ -1107,7 +1108,7 @@ def test_bytes_literal(self, testdir): >>> b'foo' 'foo' """) - reprec = testdir.inline_run(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) passed = int(sys.version_info[0] == 2) reprec.assertoutcome(passed=passed, failed=int(not passed)) @@ -1373,7 +1374,7 @@ def add_contextlib(doctest_namespace): >>> print(cl.__name__) contextlib """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) @pytest.mark.parametrize('scope', SCOPES) @@ -1691,7 +1692,7 @@ def test_unicode_doctest(self, testdir): >>> 1/0 # Byé 1 """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '* REASON: ZeroDivisionError*', '*1 failed*', From 88cd56c9e3f6cd3b0a2a51fc341809b277a46ead Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 18:06:53 -0400 Subject: [PATCH 09/31] update changelog --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e49f758c..e22123c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 0.15.9 - Unreleased +### Changed + +* Added GitHub actions to the CI + + +### Fixed + +* Fixed minor test failures + ## Version 0.15.8 - Released 2021-09-02 From f0e014f53317f456ce18354b87071203b373a8a9 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 6 Sep 2021 21:33:59 -0400 Subject: [PATCH 10/31] Update README and demos --- CHANGELOG.md | 6 +- README.rst | 14 ++-- dev/_compare/base_common.py | 6 -- dev/_compare/compare.py | 77 ------------------- .../{base_diff.py => demo_enhancements.py} | 16 ++-- dev/_compare/demo_failures.py | 26 +++++++ 6 files changed, 45 insertions(+), 100 deletions(-) delete mode 100644 dev/_compare/base_common.py delete mode 100644 dev/_compare/compare.py rename dev/_compare/{base_diff.py => demo_enhancements.py} (80%) create mode 100644 dev/_compare/demo_failures.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e22123c5..f163c2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -173,7 +173,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed * `PythonPathContext` now works in more corner cases, although some rarer corner cases will now break. This trade-off should be a net positive. -* Releases are handled by TravisCI and will be signed with the GPG key 98007794ED130347559354B1109AC852D297D757 (note we will rotate this key in 1 year). +* Releases are handled by TravisCI and will be signed with the GPG key 98007794ED130347559354B1109AC852D297D757 (note we will rotate this key in 1 year). <- (2021-09-06) lol that did not happen, dsomeday I'll get around to setting up rotating GPG keys. ## [Version 0.10.0] - Released 2019-08-15 @@ -185,8 +185,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm * Add `--version` option to CLI interface ### Changed -* Improved backwards compatibility. Explicit continuations now work more similarly to the original doctest. -* You no longer need a comment to denote that a `...` is a continuation and not a ellipsis. +* You no longer need a comment to denote that a `...` is a continuation and not + a ellipsis. (i.e. you don't need to write `... #`) * Want statements will check against return values in nested continuations * Cleaned up internal code, private APIs may break. * Failed doctests will now print their original line prefixes (either `>>> ` or `... ` when possible) diff --git a/README.rst b/README.rst index b6616acc..9a46fa44 100644 --- a/README.rst +++ b/README.rst @@ -268,8 +268,9 @@ The main enhancements ``xdoctest`` offers over ``doctest`` are: 2. Additionally, the multi-line strings don't require any prefix (but its ok if they do have either prefix). 3. Tests are executed in blocks, rather than line-by-line, thus - comment-based directives (e.g. ``# doctest: +SKIP``) are now applied - to an entire block, rather than just a single line. + comment-based directives (e.g. ``# doctest: +SKIP``) can now applied + to an entire block (by placing it one the line above), in addition to having + it just apply to a single line (by placing it in-line at the end). 4. Tests without a "want" statement will ignore any stdout / final evaluated value. This makes it easy to use simple assert statements to perform checks in code that might write to stdout. @@ -278,11 +279,10 @@ The main enhancements ``xdoctest`` offers over ``doctest`` are: 6. Ouptut from multiple sequential print statements can now be checked by a single "got" statement. (new in 0.4.0). -See code in ``_compare/compare.py`` and ``_compare/base_diff.py`` for a demo -that illustrates several of these enhancements. This demo mostly shows cases -where ``xdoctest`` works but ``doctest`` fails, but it does show **the only -corner case I can find** where ``doctest`` works but ``xdoctest`` does not. -Feel free to submit more in an issue if you can find any other backwards +See code in ``dev/_compare/demo_enhancements.py`` for a demo that illustrates +several of these enhancements. This demo shows cases where ``xdoctest`` works +but ``doctest`` fails. As of version 0.9.1, there are no known syntax backwards +incompatability. Please submit an issue if you can find any backwards incompatible cases. diff --git a/dev/_compare/base_common.py b/dev/_compare/base_common.py deleted file mode 100644 index 75aa4f52..00000000 --- a/dev/_compare/base_common.py +++ /dev/null @@ -1,6 +0,0 @@ -def do_asserts_work(): - """ - >>> # xdoctest: +REQUIRES(--demo-failure) - >>> assert False, 'this test should fail' - """ - pass diff --git a/dev/_compare/compare.py b/dev/_compare/compare.py deleted file mode 100644 index c41b66fd..00000000 --- a/dev/_compare/compare.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Compare xdoctest with doctest - -This file autogenreates two files: compare_doctest.py and compare_xdoctest.py -They will have the same body up until the main block as defined in -base_diff.py. One will run doctest and the other will run xdoctest. See the -difference. -""" -import ubelt as ub - - -def generate(): - content = ub.readfrom('base_diff.py') + '\n\n' - xdoc_version = content + ub.codeblock( - ''' - if __name__ == '__main__': - import xdoctest - xdoctest.doctest_module(__file__) - ''') + '\n' - - doc_version = content + ub.codeblock( - ''' - if __name__ == '__main__': - import doctest - doctest.testmod() - ''') + '\n' - - ub.writeto('_doc_version.py', doc_version) - ub.writeto('_xdoc_version.py', xdoc_version) - - -def main(): - generate() - # Run the files - - print('\n\n' + ub.codeblock( - ''' - ___ ____ ____ ___ ____ ____ ___ - | \ | | | | |___ [__ | - |__/ |__| |___ | |___ ___] | - - Cant do most of this in doctest, although apparently you can - use asserts, whereas I previously thought they didnt work - ''') + '\n\n') - - ub.cmd('python _doc_version.py', verbose=2) - - print('\n\n' + ub.codeblock( - ''' - _ _ ___ ____ ____ ___ ____ ____ ___ - \/ | \ | | | | |___ [__ | - _/\_ |__/ |__| |___ | |___ ___] | - - Just run the assert failure to illustrate how failures look - ''') + '\n\n') - ub.cmd('python _xdoc_version.py do_asserts_work --demo-failure', verbose=2) - - print('\n\n' + ub.codeblock( - ''' - _ _ ___ ____ ____ ___ ____ ____ ___ - \/ | \ | | | | |___ [__ | - _/\_ |__/ |__| |___ | |___ ___] | - - Run all other tests, to show how the ast based xdoctest can deal with - syntax that regex based doctest cannot handle. - ''') + '\n\n') - ub.cmd('python _xdoc_version.py all', verbose=2) - - -if __name__ == '__main__': - """ - CommandLine: - python ~/code/xdoctest/_compare/compare.py all - python xdoc_version.py all - python doc_version.py - """ - main() diff --git a/dev/_compare/base_diff.py b/dev/_compare/demo_enhancements.py similarity index 80% rename from dev/_compare/base_diff.py rename to dev/_compare/demo_enhancements.py index c07d03b3..aa4a1da6 100644 --- a/dev/_compare/base_diff.py +++ b/dev/_compare/demo_enhancements.py @@ -1,8 +1,12 @@ """ -TODO: - Anything that works in both should be moved into a different files to show - commonalities, and anything that only works in one should be in another - file to show differences. +This file contains doctests that work in xdoctest but fail in doctest + +Use the following command lines to run the doctest and xdoctest version to see +the difference: + +CommandLine: + python -m xdoctest demo_enhancements.py + python -m doctest demo_enhancements.py """ @@ -29,8 +33,6 @@ def embeded_triple_quotes(): """ pass -# TODO: fix the higlighting of the "got" string when dumping test results - def sequential_print_statements(): """ @@ -54,7 +56,7 @@ def repl_print_statements(): def multiple_eval_for_loops_v1(): """ - This is one corner case, where doctest can do something xdoctest cannot. + Previously this failed in xdoctest, but now it works as of 0.9.1 >>> for i in range(2): ... '%s' % i diff --git a/dev/_compare/demo_failures.py b/dev/_compare/demo_failures.py new file mode 100644 index 00000000..5af86dad --- /dev/null +++ b/dev/_compare/demo_failures.py @@ -0,0 +1,26 @@ +""" +This contains that fail in both. This demos what correct +failures look like in each case. + +CommandLine: + python -m xdoctest demo_failures.py + python -m doctest demo_failures.py +""" + + +def do_asserts_work(): + """ + >>> assert False, 'this test should fail' + """ + pass + + +def multiple_eval_for_loops_v1_fail(): + """ + + >>> for i in range(2): + ... '%s' % i + ... + 0 + 1 + """ From 7e7859dbfdfc12b1f237d1f0b68f94bedc3c5216 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 8 Sep 2021 10:32:05 -0400 Subject: [PATCH 11/31] wip --- README.rst | 2 +- dev/_compare/demo_enhancements.py | 17 +++++++++ dev/_compare/demo_issue_106.py | 58 +++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 dev/_compare/demo_issue_106.py diff --git a/README.rst b/README.rst index 9a46fa44..1c3e5ffc 100644 --- a/README.rst +++ b/README.rst @@ -393,7 +393,7 @@ Thus, a test that fails in ``doctest`` based on a "got"/"want" check, may pass in ``xdoctest``. For this reason it is recommended that you rely on coded ``assert``-statements for system-critical code. This also makes it much easier to transform your ``xdoctest`` into a ``unittest`` when you realize your -doctests start getting too long. +doctests are getting too long. diff --git a/dev/_compare/demo_enhancements.py b/dev/_compare/demo_enhancements.py index aa4a1da6..09540842 100644 --- a/dev/_compare/demo_enhancements.py +++ b/dev/_compare/demo_enhancements.py @@ -76,3 +76,20 @@ def multiple_eval_for_loops_v2(): 0 1 """ + + +def compact_style_code(): + """ + This compact style is a bit ugly, but it should still be valid python + + Exception: + >>> try: raise Exception # doctest: +ELLIPSIS + ... except Exception: raise + Traceback (most recent call last): + ... + Exception + ... + + """ + try: raise Exception # NOQA + except Exception: pass # NOQA diff --git a/dev/_compare/demo_issue_106.py b/dev/_compare/demo_issue_106.py new file mode 100644 index 00000000..ec298b87 --- /dev/null +++ b/dev/_compare/demo_issue_106.py @@ -0,0 +1,58 @@ +""" + https://github.com/Erotemic/xdoctest/issues/106 + python -m xdoctest demo_issue_106.py + python -m doctest demo_issue_106.py +""" +import sys + + +def logTraceback(logFunction): + r""" Logs the exception traceback to the specified log function. + + >>> try: raise Exception() # doctest: +ELLIPSIS + ... except Exception: logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) + Traceback (most recent call last): + ... + Exception + ... + """ + sys.exc_info() + import xdev + xdev.embed() + logFunction + pass + + +# def compact_style_code(): +# """ +# This compact style is a bit ugly, but it should still be valid python + +# Exception: +# >>> try: raise Exception # doctest: +ELLIPSIS +# ... except Exception: raise +# Traceback (most recent call last): +# ... +# Exception +# ... + +# """ +# try: raise Exception # NOQA +# except Exception: pass # NOQA + + +def logTraceback2(logFunction): + r""" Logs the exception traceback to the specified log function. + + >>> try: + ... raise Exception() + ... except Exception: + ... logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) + Traceback (most recent call last): + ... + Exception + ... + """ + import sys + logFunction(*sys.exec_info) + logFunction() + pass From 0fc5e0be538e2e76cae89d82983e54fb5230fbf8 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 13 Sep 2021 01:33:25 -0400 Subject: [PATCH 12/31] Working on issue 106 --- dev/_compare/demo_issue_106.py | 110 +++++++++++++++++++++++---------- testing/test_core.py | 30 +++++++++ xdoctest/doctest_example.py | 5 +- xdoctest/doctest_part.py | 9 +++ xdoctest/parser.py | 56 ++++++++++++++--- xdoctest/static_analysis.py | 16 ++++- 6 files changed, 183 insertions(+), 43 deletions(-) diff --git a/dev/_compare/demo_issue_106.py b/dev/_compare/demo_issue_106.py index ec298b87..82bc5416 100644 --- a/dev/_compare/demo_issue_106.py +++ b/dev/_compare/demo_issue_106.py @@ -1,58 +1,104 @@ -""" +r""" https://github.com/Erotemic/xdoctest/issues/106 + cd ~/code/xdoctest/dev/_compare/ python -m xdoctest demo_issue_106.py python -m doctest demo_issue_106.py + +Note: + the reason this fails is because this fails: + + compile('try: raise Exception\nexcept Exception: print', mode='single', filename="") + + + In exec mode we are ok + + compile('try: raise Exception\nexcept Exception: print', mode='exec', filename="") + + This has to do with the assign ps1 line function that determines if + we should be in exec or single mode + + Other tests + + + compile('if 1:\n a', mode='single', filename="") + compile('if 1:\n print', mode='single', filename="") + compile('if 1:\n x = 1\n y = 2\nelse:\n pass', mode='single', filename="") + + compile('try:\n raise Exception\nexcept Exception:\n pass', mode='single', filename="") + compile('try: raise Exception\nexcept Exception: pass', mode='single', filename="") + + except Exception: print', mode='single', filename="") + """ import sys -def logTraceback(logFunction): +def logTracebackThisDoesnt(logFunction): r""" Logs the exception traceback to the specified log function. + >>> # xdoctest: +IGNORE_WANT >>> try: raise Exception() # doctest: +ELLIPSIS - ... except Exception: logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) + ... except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) Traceback (most recent call last): ... Exception ... """ sys.exc_info() - import xdev - xdev.embed() + # import xdev + # xdev.embed() logFunction pass -# def compact_style_code(): +# def logTracebackThisWorks(logFunction): +# r""" Logs the exception traceback to the specified log function. + +# >>> try: raise Exception() # doctest: +ELLIPSIS +# >>> except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) +# Traceback (most recent call last): +# ... +# Exception +# ... # """ -# This compact style is a bit ugly, but it should still be valid python +# sys.exc_info() +# # import xdev +# # xdev.embed() +# logFunction +# pass -# Exception: -# >>> try: raise Exception # doctest: +ELLIPSIS -# ... except Exception: raise -# Traceback (most recent call last): -# ... -# Exception -# ... +# # ... except Exception: logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) -# """ -# try: raise Exception # NOQA -# except Exception: pass # NOQA +# # def compact_style_code(): +# # """ +# # This compact style is a bit ugly, but it should still be valid python +# # Exception: +# # >>> try: raise Exception # doctest: +ELLIPSIS +# # ... except Exception: raise +# # Traceback (most recent call last): +# # ... +# # Exception +# # ... -def logTraceback2(logFunction): - r""" Logs the exception traceback to the specified log function. +# # """ +# # try: raise Exception # NOQA +# # except Exception: pass # NOQA - >>> try: - ... raise Exception() - ... except Exception: - ... logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) - Traceback (most recent call last): - ... - Exception - ... - """ - import sys - logFunction(*sys.exec_info) - logFunction() - pass + +# def logTraceback2(logFunction): +# r""" Logs the exception traceback to the specified log function. + +# >>> try: +# ... raise Exception() +# ... except Exception: +# ... logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) +# Traceback (most recent call last): +# ... +# Exception +# ... +# """ +# import sys +# logFunction(*sys.exec_info) +# logFunction() +# pass diff --git a/testing/test_core.py b/testing/test_core.py index 03b5d522..5290916a 100644 --- a/testing/test_core.py +++ b/testing/test_core.py @@ -499,6 +499,36 @@ def test_backwards_compat_indent_value(): assert status['passed'] +def test_concise_exceptions(): + """ + CommandLine: + xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_exceptions + """ + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> # xdoctest: +IGNORE_WANT + >>> try: raise Exception + ... except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) + a bad want string + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> # xdoctest: +IGNORE_WANT + >>> try: raise Exception + >>> except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) + a bad want string + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + if __name__ == '__main__': """ CommandLine: diff --git a/xdoctest/doctest_example.py b/xdoctest/doctest_example.py index 66194b31..628d3d26 100644 --- a/xdoctest/doctest_example.py +++ b/xdoctest/doctest_example.py @@ -609,8 +609,9 @@ def run(self, verbose=None, on_error=None): # part.compile_mode can be single, exec, or eval. # Typically single is used instead of eval self._partfilename = '' + source_text = part.compilable_source() code = compile( - part.source, mode=part.compile_mode, + source_text, mode=part.compile_mode, filename=self._partfilename, flags=compileflags, dont_inherit=True ) @@ -930,7 +931,7 @@ def repr_failure(self, with_tb=True): # lines += ['{}'.format(list(failed_part.directives))] # lines += ['Failed part source:'] - # lines += failed_part.source.splitlines() + # lines += failed_part.exec_lines # lines += ['Failed part want:'] # if failed_part.want_lines: # lines += failed_part.want_lines diff --git a/xdoctest/doctest_part.py b/xdoctest/doctest_part.py index a0fcbeb0..d998820d 100644 --- a/xdoctest/doctest_part.py +++ b/xdoctest/doctest_part.py @@ -63,6 +63,15 @@ def n_want_lines(self): def source(self): return '\n'.join(self.exec_lines) + def compilable_source(self): + """ + Use this to build the string for compile. Takes care of a corner case. + """ + if self.compile_mode == 'single': + return '\n'.join(self.exec_lines + ['']) + else: + return '\n'.join(self.exec_lines) + @property def directives(self): """ diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 9eab6ab0..d077533b 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -48,15 +48,22 @@ DEBUG = '--debug' in sys.argv +DEBUG = 10 + INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) +# This issue was resolved in 3.7 +NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 7) + + class DoctestParser(object): r""" Breaks docstrings into parts using the `parse` method. Example: + >>> from xdoctest.parser import * # NOQA >>> parser = DoctestParser() >>> doctest_parts = parser.parse( >>> ''' @@ -176,9 +183,9 @@ def parse(self, string, info=None): tb_text = ub.indent(tb_text) print(tb_text) - print('Failed to parse string = <{[<{[<{[') + print('Failed to parse string = <{[<{[<{[ # xdoc debug') print(string) - print(']}>a]}>]}> # end string') + print(']}>]}>]}> # xdoc debug end string') print('info = {}'.format(ub.repr2(info))) print('-----') @@ -435,9 +442,10 @@ def _locate_ps1_linenos(self, source_lines): Returns: Tuple[List[int], bool]: - a list of indices indicating which lines are considered "PS1" - and a flag indicating if the final line should be considered - for a got/want assertion. + linenos is the first value a list of indices indicating which + lines are considered "PS1" and + eval_final, the second value, is a flag indicating if the final + line should be considered for a got/want assertion. Example: >>> self = DoctestParser() @@ -452,6 +460,19 @@ def _locate_ps1_linenos(self, source_lines): >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 2] >>> assert eval_final is True + + Example: + >>> from xdoctest.parser import * # NOQA + >>> self = DoctestParser() + >>> source_lines = [ + >>> '>>> x = 1', + >>> '>>> try: raise Exception', + >>> '>>> except Exception: pass', + >>> '...', + >>> ] + >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> assert linenos == [0, 1] + >>> assert not eval_final """ # Strip indentation (and PS1 / PS2 from source) exec_source_lines = [p[4:] for p in source_lines] @@ -513,7 +534,7 @@ def balanced_intervals(lines): statement_nodes = pt.body ps1_linenos = [node.lineno - 1 for node in statement_nodes] - NEED_16806_WORKAROUND = True + # NEED_16806_WORKAROUND = 1 if NEED_16806_WORKAROUND: # pragma: nobranch ps1_linenos = self._workaround_16806( ps1_linenos, exec_source_lines) @@ -664,6 +685,12 @@ def _label_docsrc_lines(self, string): for line_idx, line in line_iter: match = INDENT_RE.search(line) line_indent = 0 if match is None else (match.end() - match.start()) + if DEBUG: # nocover + print('Next line {}: {}'.format(line_idx, line), 'green') + print('state_indent = {!r}'.format(state_indent)) + print('match = {!r}'.format(match)) + print('line_indent = {!r}'.format(line_indent)) + norm_line = line[state_indent:] # Normalize line indentation strip_line = line.strip() @@ -731,6 +758,7 @@ def _label_docsrc_lines(self, string): print('completing source') for part, norm_line in _complete_source(line, state_indent, line_iter): if DEBUG > 4: # nocover + print('Append Completion Line:') print('part = {!r}'.format(part)) print('norm_line = {!r}'.format(norm_line)) print('curr_state = {!r}'.format(curr_state)) @@ -783,11 +811,25 @@ def _complete_source(line, state_indent, line_iter): """ helper remove lines from the iterator if they are needed to complete source + + This uses :func:`static.is_balanced_statement` to do the heavy lifting + + Example: + >>> from xdoctest.parser import * # NOQA + >>> from xdoctest.parser import _complete_source + >>> state_indent = 0 + >>> line = '>>> x = { # The line is not finished' + >>> remain_lines = ['>>> 1:2,', '>>> 3:4,', '>>> 5:6}', '>>> y = 7'] + >>> line_iter = enumerate(remain_lines, start=1) + >>> finished = list(_complete_source(line, state_indent, line_iter)) + >>> final = chr(10).join([t[1] for t in finished]) + >>> print(final) """ norm_line = line[state_indent:] # Normalize line indentation prefix = norm_line[:4] suffix = norm_line[4:] - assert prefix.strip() in {'>>>', '...'}, '{}'.format(prefix) + assert prefix.strip() in {'>>>', '...'}, ( + 'unexpected prefix: {!r}'.format(prefix)) yield line, norm_line source_parts = [suffix] diff --git a/xdoctest/static_analysis.py b/xdoctest/static_analysis.py index 80b53740..0b1297e9 100644 --- a/xdoctest/static_analysis.py +++ b/xdoctest/static_analysis.py @@ -841,7 +841,7 @@ def package_modpaths(pkgpath, with_pkg=False, with_mod=True, followlinks=True, break -def is_balanced_statement(lines, only_tokens=False): +def is_balanced_statement(lines, only_tokens=False, reraise=0): r""" Checks if the lines have balanced braces and quotes. @@ -900,6 +900,13 @@ def is_balanced_statement(lines, only_tokens=False): >>> ] >>> print('\n'.join(source_parts)) >>> assert is_balanced_statement(source_parts) + + Doctest: + >>> lines = ['try: raise Exception'] + >>> is_balanced_statement(lines, only_tokens=1) + True + >>> is_balanced_statement(lines, only_tokens=0) + False """ # Only iterate through non-empty lines otherwise tokenize will stop short lines = list(lines) @@ -912,11 +919,15 @@ def _readline(): except tokenize.TokenError as ex: message = ex.args[0] if message.startswith('EOF in multi-line'): + if reraise: + raise return False raise except IndentationError as ex: message = ex.args[0] if message.startswith('unindent does not match any outer indentation'): + if reraise: + raise return False raise else: @@ -932,8 +943,9 @@ def _readline(): # text = dedent(text) six_axt_parse(text) except SyntaxError: + if reraise: + raise return False - return True From b5d04ca8ec102b6c7d99828700fda4a48adca46a Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 13 Sep 2021 01:35:14 -0400 Subject: [PATCH 13/31] removed debug=10 --- xdoctest/parser.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/xdoctest/parser.py b/xdoctest/parser.py index d077533b..087eb680 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -48,9 +48,6 @@ DEBUG = '--debug' in sys.argv -DEBUG = 10 - - INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) From 064035fadba202d62645aa1c383ce588b5f73e4d Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 13 Sep 2021 01:36:40 -0400 Subject: [PATCH 14/31] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f163c2d6..5a58e807 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,13 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed * Added GitHub actions to the CI +* Disabled workaround 16806 in Python 3.7+ ### Fixed * Fixed minor test failures +* Fixed #106 - an issue to do with compiling multiline-single statements ## Version 0.15.8 - Released 2021-09-02 From 1b1254e98b616ab28387ea52bd02be11ae1b7aae Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 13 Sep 2021 01:42:21 -0400 Subject: [PATCH 15/31] wip --- xdoctest/parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 087eb680..c7efd420 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -52,7 +52,8 @@ # This issue was resolved in 3.7 -NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 7) +# NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 7) +NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 8) class DoctestParser(object): From b4f9eb4963892e633adcbf6ec7e32cc41b15ccaa Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 13 Sep 2021 01:48:10 -0400 Subject: [PATCH 16/31] wip --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a58e807..d60de048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed * Added GitHub actions to the CI -* Disabled workaround 16806 in Python 3.7+ +* Disabled workaround 16806 in Python 3.8+ ### Fixed From 0308196e8c836026f320973bb7f70f1b9b531616 Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 15 Sep 2021 21:27:45 -0400 Subject: [PATCH 17/31] wip --- dev/_compare/demo_issue_106.py | 10 ++++++++++ xdoctest/parser.py | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/dev/_compare/demo_issue_106.py b/dev/_compare/demo_issue_106.py index 82bc5416..397494bb 100644 --- a/dev/_compare/demo_issue_106.py +++ b/dev/_compare/demo_issue_106.py @@ -51,6 +51,16 @@ def logTracebackThisDoesnt(logFunction): pass +def slurpFile(path, mode, maxbytes, **kwds): + """ + >>> import os; slurpFile(os.path.abspath(__file__), mode = 'rb')[:9] + b'# coding=' + >>> import os; slurpFile(os.path.abspath(__file__), encoding='utf-8')[:9] + '# coding=' + """ + pass + + # def logTracebackThisWorks(logFunction): # r""" Logs the exception traceback to the specified log function. diff --git a/xdoctest/parser.py b/xdoctest/parser.py index c7efd420..7638a0b9 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -684,7 +684,7 @@ def _label_docsrc_lines(self, string): match = INDENT_RE.search(line) line_indent = 0 if match is None else (match.end() - match.start()) if DEBUG: # nocover - print('Next line {}: {}'.format(line_idx, line), 'green') + print('Next line {}: {}'.format(line_idx, line)) print('state_indent = {!r}'.format(state_indent)) print('match = {!r}'.format(match)) print('line_indent = {!r}'.format(line_indent)) From 4f56df5e0e196c6a038fbde6089cfcb41d3aaee3 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 16 Sep 2021 23:04:05 -0400 Subject: [PATCH 18/31] Refactor weird eval_final to mode_hint --- testing/test_core.py | 74 ++++++++++++++++++++++++++++++++- testing/test_parser.py | 50 +++++++++-------------- xdoctest/parser.py | 92 +++++++++++++++++++++++++++++------------- 3 files changed, 155 insertions(+), 61 deletions(-) diff --git a/testing/test_core.py b/testing/test_core.py index 5290916a..23cba1e7 100644 --- a/testing/test_core.py +++ b/testing/test_core.py @@ -499,10 +499,10 @@ def test_backwards_compat_indent_value(): assert status['passed'] -def test_concise_exceptions(): +def test_concise_try_except(): """ CommandLine: - xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_exceptions + xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_try_except """ from xdoctest.doctest_example import DocTest example = DocTest( @@ -529,6 +529,76 @@ def test_concise_exceptions(): assert status['passed'] +def test_semicolon_line(): + r""" + Test for https://github.com/Erotemic/xdoctest/issues/108 + + Note: + Notes on the issue: + + .. code:: python + # This works + compile("import os; print(os)", filename="", mode='exec') + compile("import os; print(os)", filename="", mode='single') + + compile("1; 2", filename="", mode='exec') + compile("1; 2", filename="", mode='single') + + compile("print();print()", filename="", mode='single') + compile("print();print()", filename="", mode='exec') + + compile("print()", filename="", mode='eval') + compile("print()", filename="", mode='exec') + compile("print()", filename="", mode='single') + + # This breaks: + compile("import os; print(os)", filename="", mode='eval') + + # I suppose we can't have imports in an eval? + compile("import os\n", filename="", mode='eval') + + # Or multiple lines? + compile("print();print()", filename="", mode='eval') + + # No imports, no assignments, no semicolons + compile("1; 2", filename="", mode='eval') + + + CommandLine: + xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_exceptions + """ + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os; print(os.path.abspath('.')) + """)) + status = example.run(verbose=0) + assert status['passed'] + + # The problem case was when it was compiled with a "want" statement + # + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os; print(os.path.abspath('.')) + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + # Test single import + # import xdoctest + # xdoctest.parser.DEBUG = 100 + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + if __name__ == '__main__': """ CommandLine: diff --git a/testing/test_parser.py b/testing/test_parser.py index 2a16a2ac..503d3cdb 100644 --- a/testing/test_parser.py +++ b/testing/test_parser.py @@ -166,7 +166,7 @@ def test_label_indented_lines(): ] if labeled != expected: try: - import ubelt as ub + # import ubelt as ub # NOQA import itertools as it for got, want in it.zip_longest(labeled, expected): if got != want: @@ -191,8 +191,8 @@ def test_ps1_linenos_1(): 1 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 1] @@ -206,8 +206,8 @@ def test_ps1_linenos_2(): x = 21 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 3] @@ -221,8 +221,8 @@ def test_ps1_linenos_3(): 'x = 21' ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert not eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'exec' assert linenos == [0, 3] @@ -253,8 +253,8 @@ def test_ps1_linenos_4(): 59 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 3, 5, 9, 13, 16, 17, 20] @@ -269,8 +269,8 @@ def test_retain_source(): ''') source_lines = source.split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 1] p1, p2 = self.parse(source) assert p1.source == 'x = 2' @@ -333,9 +333,9 @@ def test_parse_eval_nowant(): self = parser.DoctestParser() parts = self.parse(string) raw_source_lines = string.split('\n')[:] - ps1_linenos, eval_final = self._locate_ps1_linenos(raw_source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(raw_source_lines) assert ps1_linenos == [0, 1] - assert eval_final + assert mode_hint == 'eval' # Only one part because there is no want assert len(parts) == 1 @@ -350,9 +350,9 @@ def test_parse_eval_single_want(): self = parser.DoctestParser() parts = self.parse(string) raw_source_lines = string.split('\n')[:-1] - ps1_linenos, eval_final = self._locate_ps1_linenos(raw_source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(raw_source_lines) assert ps1_linenos == [0, 1] - assert eval_final + assert mode_hint == 'eval' # Only one part because there is no want assert len(parts) == 2 @@ -366,7 +366,7 @@ def test_parse_comment(): labeled = self._label_docsrc_lines(string) assert labeled == [('dsrc', '>>> # nothing')] source_lines = string.split('\n')[:] - linenos, eval_final = self._locate_ps1_linenos(source_lines) + linenos, mode_hint = self._locate_ps1_linenos(source_lines) parts = self.parse(string) assert parts[0].source.strip().startswith('#') @@ -489,7 +489,7 @@ def test_repl_twoline(): def test_repl_comment_in_string(): source_lines = ['>>> x = """', ' # comment in a string', ' """'] self = parser.DoctestParser() - assert self._locate_ps1_linenos(source_lines) == ([0], False) + assert self._locate_ps1_linenos(source_lines) == ([0], 'exec') source_lines = [ '>>> x = """', @@ -500,7 +500,7 @@ def test_repl_comment_in_string(): ' """', ] self = parser.DoctestParser() - assert self._locate_ps1_linenos(source_lines) == ([0, 3], False) + assert self._locate_ps1_linenos(source_lines) == ([0, 3], 'exec') def test_inline_directive(): @@ -534,8 +534,6 @@ def test_inline_directive(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] - # print(ps1_linenos) # [0, 1, 3, 4, 7, 8, 10, 11, 12] # assert ps1_linenos == [0, 2, 5, 6, 8, 9, 10] parts = self.parse(string) @@ -563,7 +561,6 @@ def test_block_directive_nowant1(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -591,7 +588,6 @@ def test_block_directive_nowant2(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) # TODO: finsh me assert len(parts) == 2 @@ -608,9 +604,7 @@ def test_block_directive_want1_assign(): >>> _ = func2() # assign this line so we dont break it off for eval want ''') - # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -634,9 +628,7 @@ def test_block_directive_want1_eval(): >>> func2() # eval this line so it is broken off want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 2 @@ -653,9 +645,7 @@ def test_block_directive_want2_assign(): >>> _ = func3() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 2 @@ -672,9 +662,7 @@ def test_block_directive_want2_eval(): >>> func3() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -705,9 +693,7 @@ def test_block_directive_want2_eval2(): >>> func4() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 4 diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 7638a0b9..63236910 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -139,6 +139,7 @@ def parse(self, string, info=None): >>> doctest_parts = self.parse(string) >>> # each part with a want-string needs to be broken in two >>> assert len(doctest_parts) == 6 + >>> len(doctest_parts) """ if DEBUG > 1: print('\n===== PARSE ====') @@ -256,9 +257,9 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0): if DEBUG > 1: print(' * locate ps1 lines') # Find the line number of each standalone statement - ps1_linenos, eval_final = self._locate_ps1_linenos(source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(source_lines) if DEBUG > 1: - print('eval_final = {!r}'.format(eval_final)) + print('mode_hint = {!r}'.format(mode_hint)) print(' * located ps1 lines') # Find all directives here: @@ -308,7 +309,7 @@ def slice_example(s1, s2, want_lines=None): example = slice_example(s1, s2) yield example s1 = s2 - if want_lines and eval_final: + if want_lines and mode_hint in {'eval', 'single'}: # Whenever the evaluation of the final line needs to be tested # against want, that line must be separated into its own part. # We break the last line off so we can eval its value, but keep @@ -321,17 +322,17 @@ def slice_example(s1, s2, want_lines=None): s2 = None example = slice_example(s1, s2, want_lines) + + # if mode_hint is False: + # mode_hint = 'exec' + # if mode_hint is True: + # mode_hint = 'eval' + if not bool(want_lines): example.compile_mode = 'exec' else: - if eval_final is True: - example.compile_mode = 'eval' - elif eval_final is False: - example.compile_mode = 'exec' - elif eval_final == 'single': - example.compile_mode = 'single' - else: - raise KeyError(eval_final) + assert mode_hint in {'eval', 'exec', 'single'} + example.compile_mode = mode_hint if DEBUG > 1: print('example.compile_mode = {!r}'.format(example.compile_mode)) @@ -442,22 +443,22 @@ def _locate_ps1_linenos(self, source_lines): Tuple[List[int], bool]: linenos is the first value a list of indices indicating which lines are considered "PS1" and - eval_final, the second value, is a flag indicating if the final + mode_hint, the second value, is a flag indicating if the final line should be considered for a got/want assertion. Example: >>> self = DoctestParser() >>> source_lines = ['>>> def foo():', '>>> return 0', '>>> 3'] - >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 2] - >>> assert eval_final is True + >>> assert mode_hint == 'eval' Example: >>> self = DoctestParser() >>> source_lines = ['>>> x = [1, 2, ', '>>> 3, 4]', '>>> print(len(x))'] - >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 2] - >>> assert eval_final is True + >>> assert mode_hint == 'eval' Example: >>> from xdoctest.parser import * # NOQA @@ -468,9 +469,20 @@ def _locate_ps1_linenos(self, source_lines): >>> '>>> except Exception: pass', >>> '...', >>> ] - >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 1] - >>> assert not eval_final + >>> assert mode_hint == 'exec' + + Example: + >>> from xdoctest.parser import * # NOQA + >>> self = DoctestParser() + >>> source_lines = [ + >>> '>>> import os; print(os)', + >>> '...', + >>> ] + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) + >>> assert linenos == [0] + >>> assert mode_hint == 'single' """ # Strip indentation (and PS1 / PS2 from source) exec_source_lines = [p[4:] for p in source_lines] @@ -530,6 +542,9 @@ def balanced_intervals(lines): syn_ex.text = line + '\n' raise syn_ex + # print(ast.dump(pt)) + # print('pt = {!r}'.format(pt)) + statement_nodes = pt.body ps1_linenos = [node.lineno - 1 for node in statement_nodes] # NEED_16806_WORKAROUND = 1 @@ -543,17 +558,25 @@ def balanced_intervals(lines): } ps1_linenos = sorted(set(ps1_linenos).difference(ps2_linenos)) + # There are 3 ways to compile python code + # exec, eval, and single. + + # We almost always want to exec, but if we want to match the return + # value of the function, we will need to run it in eval or single mode. + mode_hint = 'exec' if len(statement_nodes) == 0: - eval_final = False + mode_hint = 'exec' else: # Is the last statement evaluate-able? if sys.version_info.major == 2: # nocover - eval_final = isinstance(statement_nodes[-1], ( - ast.Expr, ast.Print)) + # Python 2 overhead + if isinstance(statement_nodes[-1], (ast.Expr, ast.Print)): + mode_hint = 'eval' else: - # This should just be an Expr in python3 - # (todo: ensure this is true) - eval_final = isinstance(statement_nodes[-1], ast.Expr) + if isinstance(statement_nodes[-1], ast.Expr): + # This should just be an Expr in python3 + # (todo: ensure this is true) + mode_hint = 'eval' # WORKON_BACKWARDS_COMPAT_CONTINUE_EVAL: # Force doctests parts to evaluate in backwards compatible "single" @@ -561,9 +584,24 @@ def balanced_intervals(lines): if len(source_lines) > 1: if source_lines[0].startswith('>>> '): if all(_hasprefix(s, ('...',)) for s in source_lines[1:]): - eval_final = 'single' - - return ps1_linenos, eval_final + mode_hint = 'single' + + if mode_hint == 'eval': + # Also check the tokens in the source lines to look for semicolons + # to fix #108 + # Only iterate through non-empty lines otherwise tokenize will stop short + # TODO: we probably could just save the tokens if we got them earlier? + import tokenize + iterable = (line for line in exec_source_lines if line) + def _readline(): + return next(iterable) + # We cannot eval a statement with a semicolon in it + # Single should work. + if any(t.type == tokenize.OP and t.string == ';' + for t in tokenize.generate_tokens(_readline)): + mode_hint = 'single' + + return ps1_linenos, mode_hint @staticmethod def _workaround_16806(ps1_linenos, exec_source_lines): From b375a970123a8be6b155a8c928c79354b877cbfa Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 16 Sep 2021 23:13:35 -0400 Subject: [PATCH 19/31] Update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d60de048..bea2947a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Fixed * Fixed minor test failures -* Fixed #106 - an issue to do with compiling multiline-single statements +* Fixed #106 - an issue to do with compiling multiline statements in single mode. +* Fixed #108 - an issue to do with compiling semicolon token in eval mode. ## Version 0.15.8 - Released 2021-09-02 From 178cc58d2352a395538131c6b61f595b83fe5f09 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 16 Sep 2021 23:55:02 -0400 Subject: [PATCH 20/31] Fix python2 error --- xdoctest/parser.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 63236910..de69f192 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -39,6 +39,7 @@ import ast import sys import re +import tokenize from xdoctest import utils from xdoctest import directive from xdoctest import exceptions @@ -51,9 +52,9 @@ INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) -# This issue was resolved in 3.7 -# NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 7) +# This issue was resolved in 3.8 NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 8) +PY2 = (sys.version_info.major == 2) class DoctestParser(object): @@ -127,6 +128,7 @@ def parse(self, string, info=None): directive is still in effect. Example: + >>> from xdoctest.parser import * # NOQA >>> from xdoctest import parser >>> from xdoctest.docstr import docscrape_google >>> from xdoctest import core @@ -568,7 +570,7 @@ def balanced_intervals(lines): mode_hint = 'exec' else: # Is the last statement evaluate-able? - if sys.version_info.major == 2: # nocover + if PY2: # nocover # Python 2 overhead if isinstance(statement_nodes[-1], (ast.Expr, ast.Print)): mode_hint = 'eval' @@ -591,15 +593,19 @@ def balanced_intervals(lines): # to fix #108 # Only iterate through non-empty lines otherwise tokenize will stop short # TODO: we probably could just save the tokens if we got them earlier? - import tokenize iterable = (line for line in exec_source_lines if line) def _readline(): return next(iterable) # We cannot eval a statement with a semicolon in it # Single should work. - if any(t.type == tokenize.OP and t.string == ';' - for t in tokenize.generate_tokens(_readline)): - mode_hint = 'single' + if PY2: + if any(t[0] == tokenize.OP and t[1] == ';' + for t in tokenize.generate_tokens(_readline)): + mode_hint = 'single' + else: + if any(t.type == tokenize.OP and t.string == ';' + for t in tokenize.generate_tokens(_readline)): + mode_hint = 'single' return ps1_linenos, mode_hint From eb02601e061d9ede8052008d8023602c5dd3dd30 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 21 Sep 2021 20:50:42 -0400 Subject: [PATCH 21/31] Update secrets --- .github/workflows/setup_action_secrets.md | 12 ++-- .github/workflows/setup_secrets.sh | 69 +++++++++++++++++++++++ dev/ci_public_gpg_key.pgp.enc | 51 +++++++++++------ dev/ci_secret_gpg_key.pgp.enc | 18 ------ dev/ci_secret_gpg_subkeys.pgp.enc | 40 +++++++++++++ dev/gpg_owner_trust.enc | 17 +++--- dev/public_gpg_key | 2 +- dev/travis_public_gpg_key.pgp.enc | 16 ------ dev/travis_secret_gpg_key.pgp.enc | 18 ------ 9 files changed, 161 insertions(+), 82 deletions(-) create mode 100644 .github/workflows/setup_secrets.sh delete mode 100644 dev/ci_secret_gpg_key.pgp.enc create mode 100644 dev/ci_secret_gpg_subkeys.pgp.enc delete mode 100644 dev/travis_public_gpg_key.pgp.enc delete mode 100644 dev/travis_secret_gpg_key.pgp.enc diff --git a/.github/workflows/setup_action_secrets.md b/.github/workflows/setup_action_secrets.md index 6ee032a2..b0167627 100644 --- a/.github/workflows/setup_action_secrets.md +++ b/.github/workflows/setup_action_secrets.md @@ -113,7 +113,7 @@ and then a secret file that looks like this export PERSONAL_GITHUB_PUSH_TOKEN="git-push-token:" ``` -You should also make a `secret_unloader.sh` that points to a script that +You might also want to make a `secret_unloader.sh` that points to a script that unloads these secret variables from the environment. Given this file-structure setup, you can then run the following @@ -135,19 +135,19 @@ echo $TWINE_USERNAME # See previous CIRCLE_CI section for more details # HOW TO ENCRYPT YOUR SECRET GPG KEY -IDENTIFIER="travis-ci-Erotemic" -GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}' | tail -c 9) +IDENTIFIER="Erotemic-CI " +GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') echo "GPG_KEYID = $GPG_KEYID" # Export plaintext gpg public keys, private keys, and trust info mkdir -p dev -gpg --armor --export-secret-keys $GPG_KEYID > dev/ci_secret_gpg_key.pgp +gpg --armor --export-secret-subkeys $GPG_KEYID > dev/ci_secret_gpg_subkeys.pgp gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp gpg --export-ownertrust > dev/gpg_owner_trust # Encrypt gpg keys and trust with CI secret GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_key.pgp > dev/ci_secret_gpg_key.pgp.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc echo $GPG_KEYID > dev/public_gpg_key @@ -155,7 +155,7 @@ echo $GPG_KEYID > dev/public_gpg_key cat dev/public_gpg_key GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc +GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc source $(secret_unloader.sh) diff --git a/.github/workflows/setup_secrets.sh b/.github/workflows/setup_secrets.sh new file mode 100644 index 00000000..118b1e8b --- /dev/null +++ b/.github/workflows/setup_secrets.sh @@ -0,0 +1,69 @@ +__doc__=" +Development script for updating secrets when they rotate +" + +cd $HOME/code/xdoctest + +# Load or generate secrets +source $(secret_loader.sh) +echo $PYUTILS_TWINE_USERNAME +CI_SECRET=$EROTEMIC_CI_SECRET +echo "CI_SECRET = $CI_SECRET" + +# ADD RELEVANT VARIABLES TO THE CI SECRET VARIABLES + +# HOW TO ENCRYPT YOUR SECRET GPG KEY +# You need to have a known public gpg key for this to make any sense +IDENTIFIER="=Erotemic-CI " +GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') +echo "GPG_KEYID = $GPG_KEYID" + +# Export plaintext gpg public keys, private keys, and trust info +mkdir -p dev +gpg --armor --export-options export-backup --export-secret-subkeys ${GPG_KEYID} > dev/ci_secret_gpg_subkeys.pgp +gpg --armor --export ${GPG_KEYID} > dev/ci_public_gpg_key.pgp + +# Encrypt gpg keys and trust with CI secret +GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc +GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc +echo $GPG_KEYID > dev/public_gpg_key + +# Test decrpyt +cat dev/public_gpg_key +GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose +GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --list-packets --verbose + +source $(secret_unloader.sh) + +# Look at what we did, clean up, and add it to git +ls dev/*.enc +rm dev/*.pgp +git status +git add dev/*.enc +git add dev/public_gpg_key + + +_test_gnu(){ + + export GNUPGHOME=$HOME/tmp-gpg-testbed3 + mkdir -p $GNUPGHOME + ls -al $GNUPGHOME + chmod 700 -R $GNUPGHOME + + gpg -k + source $(secret_loader.sh) + CI_SECRET=$EROTEMIC_CI_SECRET + echo "CI_SECRET = $CI_SECRET" + + cat dev/public_gpg_key + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc + + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import + + gpg -k + # | gpg --import + # | gpg --list-packets --verbose + +} diff --git a/dev/ci_public_gpg_key.pgp.enc b/dev/ci_public_gpg_key.pgp.enc index f99f82ca..40eb988e 100644 --- a/dev/ci_public_gpg_key.pgp.enc +++ b/dev/ci_public_gpg_key.pgp.enc @@ -1,16 +1,35 @@ -U2FsdGVkX1/NUJtlVaXysoiU5E0+zVqAA7zqbXzkF0IM7XgzVP2h7yWJ3GE9zD23 -h9RQpwi8Vl5bSOvZV4n/lOEnhSV2pelHZOGet80ewbaKeU+Trp8Gv3pU/l9FZrAx -EU0jk5oEojHnFnC0H/uAC3qeQ8WWmpGXklpPdbMu2NiOhDFrAhAMIDkNxL5L7Rtt -DnqO5em/6iCatkTCeA9pQU7/vjV3OHNzf9XjjBcq+bOPge1hup1gJ3/TpC1ughr8 -2cYoXNaDkXnktA3NU33S94VVAXYpQVH1vMS95NBCjeru//ktf+QUv/4TuXxTU899 -N/5sQH/OP8qs+/h3+/wG6ea+A6j8G4+sgAp4HSkxazuu0zUvufjciuYg4C8lPZtl -/2GiEF78whVAgReeGvlYA7hOfGDNXfvhLKRN4QK5vE2QmJMQW7nvh0arvaBm9QCa -CaTTFf34AQkDmBounuBn0tb3bJQlaZJ1yZQhstsOJfvLJUKPHKZmIvO6mfNcaeLL -qFDCN3VU+z0IpuFucS5kr0A3IyuVr38B/E4rw9ilwfwJfbPrAI6imxa2HVXUVzWi -eOEFIlF5TFgMnY5+yKUn8YbpcVZRUbI0wuFcYFZvKDHlRMP8FibBdphyfsTujM84 -inTBMy5NJoaDDnVUShQUuiQovxGJSdttFLAoQPaUpqs9E7l5a+4pdcB/jametlgZ -ky21IogZsuQ8Hzrp2tZDk0P64qu3vCse2/hgNsBrewIkT5OpT47ZKKSzcjZEl+Od -/1Ok2glPc9Qg6lwuQT0knHVPAyl1K36ejdLHHibUiKnGXxXr1wq52OA/wmFCVzxs -J46NcSVYAOsmqp7lKWYDyHgEDnBmBAM3oyAL5rxniFhCxUBlGW0IGUBpdMR0IDr9 -btaWF/1NCJIFTExmaH+9WuFPbkLvmSKUdwIvlfzG2I5RskIxg1zRtNIO9BoQOd/r -2e9zO+zVgDBVXaBCDBtfA1+s0dtPkOnlE4O1WBP8ShqDQvs5IcmXYXJPPMiSikg7 +U2FsdGVkX18PbC4mZJwLbtMm65zwvM+KN4w7GV7m4kwm2+SS/hMmnUQzSDJAJCD5 +eA9Uix3Pw8/slqv63euDewhyuTminDNal3xHu+YhbG+nwoKfBYvRzL1Qr08mdc7L +0bjFVDMc6I42BlxWj8AOmS2sRPe6jgA3oiULKOYFnaF9bRYkFKiUTQEEwwXURiSr +BkXmRAYGcFmJZF5ylV3thSjjiGPvtuIek/qboZ29L7HY7OLPKC38f+59hoF/48yH +Wb2RhWpYakMuyMiziW44w5nQ66rFyT6Rcl5JtmUOZFNbHZd5NxeCzVgOqQwVSxNu +gTMjchrvQ40jwwA1lqO48+cSkzMA+7+MmqBDmVkpJdY9OBmrpkkipNFKn0zHqEYK +VmsBfiEUW6wuebaZMAwwsmmgGDa6d+JOFyzLx+za//OYuxOM6P3waBrnkuQyk632 +Q5NPjILq3ty9CL7CW94qK2/PW0fUJVUBgI61ljq0wfgGtoolL0WAD4DhCt7x3hdJ +bt03+CgdhI7QDnbt7XuBz68R5YyuP/2An9APkFswZXNyYjbqdQMJnjYzrVYAg2an +G+6Bc5o32B2IHfctii5nDiIPPb1ZQlha8SJl3JPYOpVT92rZ10jES+P5baBdm8tY +zShOsGJ8ecNzZQbqhb1oRW/J2cFrTfSiRRF0f9NBHWbARpO3+uF+pAXQluJMSZ86 +a1oP1LrnY/7YwqMR1AVi6zbpyefJ7jEAZNwLD1n36x5iAGJQ8ND56rQYbalBeh2o +ToY+Lmy/hAp1EiVVdQYTnoGM//zD/L29M77hL5Ft+mKUOhfBn3DlDbiMKwsgWT6G +MHu7oXynzl5CpuOEuN59Bx15yjycLMnDuBoYv8zeb8RsbwlvHqsR39mreKb4m1FE +iDIDkee5Vgxy/7ChCyDOtMcXqt7k40F4a/F/bNkya2OLHMHw6zULCBb3gPcvubS/ +Zw5man3QHMqKVjWm4MN5pR+8F0GytOR0bizIu+8r0vHZzZZE/PVzpWMf+8R9MOqe +cckxCmjmgth/4fmB8qn4fmnZc9AUWj1uOWROD0L5smNBpgfpyJgv3XMsPuBBlPP0 +Q4xntrcSmO57BTcbgP6VL3PN5sWllH3mJJQ2vyzypstFHH1Sx/GftokDDlZGUqzp +8PhM+Jb5UhrWRh0BV1CjNx2FCMXZnDrUFhkDxrFC1H1J6liXk4Oa8pxRjfOfhh+7 +BlrJmrrY5OStdqclgz3nkx2hvHE+mMrxA1IelBeqR3c31ZDgnpFR3/ifDSKAhPSr +Ub71GtmGEJDINbhFt8hvpJa5cUzszxY2s6aPKEBxtVGYI24UVNuPucHinChaJohy +yZ4m7Qh9AwHF4hqjEtaghTqRab9dJLcEuJ9961uHoigg6fTmr270bVXYQdPgSPHx +LHYz7gnKz8CONA1Le7viYEHZVJFV8fXQ820kS4rpfmICarh4u0Z2NLv23LYFzm2o +dJQcD0TZZKZunAwPJgDH4CRRUvqyAee2vZOEB5zVCH54OhizfPmwpfREgknstC2Y +5SKVCCQRX4uZmgDKnUZvPfICoyw0N9+uodEhSsTw3XI19jfbBzPlN3q29v1ZrYXh +FyuP3K59XACPSqSqyeUsipqoCtVA7V+tDb08JUzQfsvwoo9W8/q4EGe2TkxHQrR+ +tK2HOlNwj6dG8j2aw6vHKgq6cwQEWZ2tyBohnwoM1cDOWBrdore1tnazLPSpWWTX +qaWIr4u5PoDMSeTY6fdsK+xC10Y8FazRfcBehc4ZSbw8VqOrWQ5LJe7mEJJLmnRu +yv9KhwTVwhGO9DaX1OdyxpRscH6tiN7OSDGjg92c9wcTdzc8qozKocl/nHIowblW +RI0qex2UMKIhJZyMcvLxKmvax+bgplc8pMOdS5U+Pe2c5eUJmmBAXdwi6gQW8Slo +zbNMLwtQu6Ma6UiptikI1o6XLTL5i6zhWEHueEx0bdaor8vbfTCQe5a4bN/LKLs9 +T9/QtzSpS5bg7qgoZA+rFnKDLyRiaIY7YwNpb2JbdQpkhd00c/JasyZBl5R1xiZm +DngvTn1TjaAetwsunmRPH5ZE0SYM8VZ23sYQc9x0SJU6GjLgQeViU2pfv6j3KjSE +qkNRZ4yx28jObYPVVJhQweCH1RbdxpQvwSFq5AOhmYWzh6r49S/CAE581hfmi2x8 +QktYOrBGC/i18I/uzRIBCg== diff --git a/dev/ci_secret_gpg_key.pgp.enc b/dev/ci_secret_gpg_key.pgp.enc deleted file mode 100644 index 9cfa93a8..00000000 --- a/dev/ci_secret_gpg_key.pgp.enc +++ /dev/null @@ -1,18 +0,0 @@ -U2FsdGVkX1/cIIJK5fbB8daDmOnQwrhsHJjiSk5hv92Z/WKR3zTe6KG9Sktd9RTK -gVSoMBbPq/tJ5e6i0niTjouAvjdJTPP2Uc6nupduV5L/Qw5MdeQEz66eoj6e1Y5T -TIjs61LEt7AbIMnO3ohBpKEBzbIlsAYYJ7q8N4FaWLw9rLImpcibWCXW6YYiSbPI -mFbP1GMhmNqKKSV5ccMddGQjYk22DI2uSVWaV2xz82wbil02wcw7AJfD+5hoK5i+ -7Q9vBAbtt1RCw+P+O1JQ0cAmbBoMCZkL0CH5CIlnncCHf/l8vHUYppwp4fUcueW4 -AXlrVNuSdJW26jBn9YcSgWjCrtO7Uv1qZwwjz4YAaGJD0nmSIiQweXvoa3CgQrx7 -r/bln8W92vNmBy2kPmUbA1Ory3MTkw5k9+1RFnxPF+Z9OOzZW82Abt+yKW5HsA5D -fzNgfOsMY8r8ydH+oqf+sWJMO4xEZW0LZYTyZ7HiuDjWhHQABFnTFteM1eBM02yg -SzfpfJMQukf0qV8eVvgI2UjiTuengQwaAr4GPF7gs3dIZhNdZS2+zKCBIQR+fe4a -86ORIqwJxNxhwchzH47cDxDEI3By8TOCyQHZGAiOTeehdmbs9Ay+qJuWjOnUwh36 -1ACraJuMa+6btq11G2rMKNPqhQw6mGONUKPNKvYDL6lGzwUXD+WxYihFy01WnB7J -ot6wUYZDno3ut5IczcIQMR4T/D06JdGJSCsjbashTn3772Xv1uwGQ5WEoeGalcJt -OvidnnVODoPgRL03hKoeM41bwePoZGDVOg362vcw1ZlXIocnTq0tpxFaDcXU1+OP -OiqXXz62xOEMhKqDVsUU1pjG3m4pudNWy0zJEbMtt/Z1gH3mJZas7DaFsHEA53S1 -yvIN0b3uESYUlOAiRv7Fz3zx09HtPtO42anamyuCA9zdaru2msISj8ORTq7g7dH8 -pULT7K9WmXcxi3XpCan9FcQTQJCcFvOrL7jAW+BisP5RFgeH2tM5FviSI+95u1B2 -Uh00XC6qT+NaHgzJS9hvajouojV3GjdTocUSHby+5P8at4CWNiwDHlBlrusI5vb7 -XAYVynRrQVKGtnnjUHRhWJr9P89MYHx+NIaI2tnOxWRrO2AT/50dkp7cru6gh0rh diff --git a/dev/ci_secret_gpg_subkeys.pgp.enc b/dev/ci_secret_gpg_subkeys.pgp.enc new file mode 100644 index 00000000..e4cfaa73 --- /dev/null +++ b/dev/ci_secret_gpg_subkeys.pgp.enc @@ -0,0 +1,40 @@ +U2FsdGVkX19mtCufSuMCTKwPlW3lZByYR4s/O8BcJCjIwXL6Zp7wKhgIfsSGVIHp +wo9elAW34nB2j5T4qRlLMfMWxaDuvTcM1Z5jmm/5hlaShgi5X/XLdyCwHc6qR7/X +cvJmvXzKqAwWyXoDy7jyFxShvgs1Fk1q21UnEyQNqag5ik05krC8eUtmgSpUosK1 +PPObGs5xFKALylw/N0VgzmraMVCkApxAshhTuMosUmyIEJnY/Q8DdaXcF4624RHU +hAVD+7XiM0vm6avu8PGqDxYwbLoGeUAcTPouhLpNuUO3N7TCAyceUJoeEE4ulJdA +OLrtYTWjSTxzQq9o7TdRv2l6/UobMrhwn7Jk55/UTJJVMGheEojL6tx6abUt1oMy +8pFN8M6mKvPMNzE2yhAJl+PJNMTPc1HFYtapHExtIizwrxl/IptxPxoMqDHxGjYJ +3EwOrBA1du6eU8XxBrZRXWQMciECBqnidVHrUBnoiaN3Bt+t+1m75hla1L/djQxQ +qq7uckwnh2gphgS/iGOOxaeQ69vcHky4NnBQpoAB/U4PVQNQcGvsrW/DU4wcb3kx +jslz6X/ujfB0B53ANDXsHxin5EJ1p2ldLZunFyl2HVWLNhg7XQXvh1/fZKCJtCbq +LqsmgCSE0YKbDqtFS/qh+LXZCabzHuFMwrPSFIe5QIBSImgU+UblgOkBY03zB8nE +CN7X2KglovpkWdM3T/nU+r3Utgv/n5PStqCs9JVan5o2Op2idUGZUcuvbJ5AZR9A +5eTEbPg4jMeXzJAZKGqop1yL4Z7fD/3DpG2MDBRPTuB4WuDhR4wKvNubTX1V6dJQ +AItNjQ8VN9DtBpJz4cmtpGju0h1tHRLIVxvlBNfm68diFM7ZsArypIy6DnBraEAH +xnvktO716TuGb1Mv3pm1GwKKjXuXWHV1/6Qd/N6ehvSrkCmmlBM6WiGZYvnllj/X +XglaJ5CDgnG/OmXkQWFZk9eYTHnTeyITVTaT4zG95AVqjePRblkwwCy5fv5R3Y14 +w/zaGxfS4VUoZBIBRAsnjLybKpkhsz5BvXr0wLYoD/RyNk/6xRuxD+FSC8bx4pe4 +zWdJeGg4kcHremMEtSwcQClOBgTlHHWBNhIpi6GjhlMbaq5E2xwiaFzxdOqN0aDR +VqnvklQGaWWunIwLkY4WJyGuNH5lehffRIFscT0rsLtAi/t/PcU8Oh25+078Ed+/ +7eJM5JjIHMXNCEmxN60Y4eNRYfe2FQUOoxIfjv1IvmtEegIzj5Mdc4ZDzgNjLmYp +MqRotXtDBZ1pku0fnHwjiveDzzT5MzTX3GtV2Grx7t38YeT3fJR/r1P4dDJfdKGR +1ukq052lmRdiIvUJEE0ViB+uYu3mngywDpaggtJgcjiVqaytdAjcjBuD+0zMAckQ +vJ6TTVdr4chjEbQXPZLbAduQMtgTCwrJThf4ZUSIym/YNXky24CyLFRwWaLV/oIE +4ixuqWXq2DGO+GYEUECNEb0+IEX8/0l6mvfn9FUF9+4Z7zvdgi2m1/jIcpz/aK+J +0lQ72ZECDd+yXffxbGLOVtUmk9x7MT/6alR11C4ll7YfZAP/GBGvOqjpHa3gsVPP +dK4fXRi+LQH40ZEsJ6ByZazG40Pyhr8SAK/VoXMJ65jTDFkoIAgkU6AWAwebE7LM +ys8KiKIh3YvST1ZBe29/CDb72JHgT8fxzFkVRmb1E9vuHz3eTsgEZYEuqCxLyJGA ++EbudFiTJNlCXv/V+CkJAk8T1C5ZoYIAM27k7Hi8RT8oM6gU3FfPnCzhaw50+rsX +xpX/HBA1Hz346U9IPGWLlbRJeQZprMFeI78oTL4U5kXFb0+RgZEWVpyyViqb+ka4 +5DL5i+K5Vf5FFOs23FkqTY7e+8tFfne0le1pVBi9CHztKFlmQ+vM+Ctv9HJVVIGS +f/w30wv9T1o91yYSf1y1G0spZnT8L0RtSx7EKc5QHaSrK075NAhgO4dSAs1R4twv +tyXZ0l+ztJ4jzo6ldFsQIdjyb1oAC3bL8qDfAJYn/pC31+30kHth9ZMv5XxUAxin +/x4/VpIIwBdUfQVSxTNMMbLxMK7cbIKQKvYvqfeJC0KTVJ3CDC8dKAh4NBdApRIV +gk6O7tyuIU8n+FdH9EtEePcOmoTkiGBqr2zg84W1D8/8rbCeAAgwY4b1tiVhwrFd +czVp07eblU953fhkrJtwOfJDxZlPtzTv7DWI1jJbPjUWkv7lSQoSZdAuQLQfSobs +nOqL58LrVtMuuHk5f31zx3yGpKudyZdxLtcQ9Fth8gJ/cLjG15dX75tPciv3j2Cv +BLKjWA4Hw0nDrC6A3qEteMmFFvCxuawS0JxaBNQg9zRLwz9NPZ5mAku72NtM4r0A +bdt+SrjsnEyvRAK7TZikVHhOy/aiNO56bST0O1eJjBchRyKAtUPwpijgWpXIcWqB +xf9/IJfJMpm6wiKBeQeNQPtEJwuaS3P73cYAy2nycZbBjuI+mtcVRe8daIVMnPBy +bktPQNqYHPFPRBV96lRPd/FTAzQrW5sjPdnPM7mp86EC6/GXCOExaYP3a1aY8LXP diff --git a/dev/gpg_owner_trust.enc b/dev/gpg_owner_trust.enc index ffe6d69c..7c8e6fcf 100644 --- a/dev/gpg_owner_trust.enc +++ b/dev/gpg_owner_trust.enc @@ -1,7 +1,10 @@ -U2FsdGVkX19+WoDVlFCHUeo0irIBBe8Sp9N1YAi5DhAkKYd6M+C1YnwcSjQJaLvj -eM+HWcyBZfj1mN537jMmnrgS7cxGh4QvMDBMVifw1UYMGCcn/Rf+b2+0BymcGuLo -c3JEmdFXjIw2a0eHh8seBFq+HwLoU3bymz7Lr0XQK7Y6hPPSm7Cwo1HjOwuDXFa1 -RIutCuQC79og+sryYfQPBTEiCeJW0j0maUpS9H5gfnqXafp1+Fg7if/AskuATMXG -dUtkIYFJXwgY5qvD1pIQnmC/rKAZj5o8x8u2XewgVX1PHk6nrBI1dBUMnkRs2h8N -p6dxj+yC68YSUpdlPIvLdWxihOLjIlUbkdlpIFSO7hNbuKfb22On/TFHMKYeqN7Y -IGdcNCFiFaUkZWiTI6Nt6/6HgZNFrhqVjtGZDV7pcwM= +U2FsdGVkX19u3udL4wxqGFCI5wmjZgfGufyvYlKI54O8zYZDFEHwY6RCMBTBELSv +G8RB+sZNmzhCLsyhgAT1QRRnKpYhetf3fw0HB9tnr1SqHdwzpYlUw6yUVBQOpasx +WhOXhN/1lOWNZqfNY1T6WarXXXO3mpGjpT6G0Mrf5SneRaemAZH9TgqiwlthpWqF +IjpvlqyCS6YbtXWNTb+A1pm72dmi8zcqrD7WAro0sHF+WtC6xE+WOEmUB52u0RXN +PMGK5lzv22iy1aFRVb29GYOtQddGDwIDnGLwwkt72ft+0qjfZVokyziaJdAss4MU +YuyxAwS0DAhBfu6BjW8TtBB7uFEKHABm+HyEi4W72e1ZTbylv/H9TXXzofhbNAPS +2Pn4axMy50zeIZeZHwi6fdQbMVfz2vamWhA2et04H+oiGnSMSorXCgZAWqYboM+I +2aCbUc9/I+Rju08p7gfJaCLhmEkRqD0RwjpqX50xyXcIcHA+9zpohEwGcRmnkZfJ +LHvNScvuSJum8EE6pCn+3WBaXOGGgEbooWzbQ/fHgMGjqc9yiVrcPL4s3NNMQNtl +3D8+cYYmk53UgW7yRsWdyw== diff --git a/dev/public_gpg_key b/dev/public_gpg_key index debca485..91ee9e69 100644 --- a/dev/public_gpg_key +++ b/dev/public_gpg_key @@ -1 +1 @@ -D297D757 +70858F4D01314BF21427676F3D568E6559A34380 diff --git a/dev/travis_public_gpg_key.pgp.enc b/dev/travis_public_gpg_key.pgp.enc deleted file mode 100644 index 7fc3cfed..00000000 --- a/dev/travis_public_gpg_key.pgp.enc +++ /dev/null @@ -1,16 +0,0 @@ -U2FsdGVkX18leu1FqCbf5paHmrfkdtHWuu8lmIx5DEVDOYVeJvwJG4DtBGUzTw09 -LNayN8/sUPDHQ//A7mgDz4jVIUr+iTVZ2h2RkydSHG6qVsJHGY6dBg8tSZ/qiKcA -r9YsLJBJZms7beUjCwG9nzTMYmRJuQ+DYjTWd4njCBdH00jEZ80hM02YAuOV1ZqQ -wY0S7VitJiyHQC6rsdNMNlViXy49iAoqCYMJyd7cbgAbj2w3QPxGlAWRcbBJ1EcK -hdbuaXdpmx69hQ+d+w1xYiptZufcDQ6hqq40sEzAlnC2ZhwqA+8GUk6whN+5foF9 -U4D4JTSyvbTNXHePGNwMXC2J9F0momKGEPG5JdJO7NLP/E5VLbtzzoDF2tFUD9W0 -vPndpfj3DxPnRJwWag8i3jHOHkrfeSM4kMNUQNyrg6pEKwvw1kVtMi6ATCvdYESl -3OPvcSlSEzCudsBdNxQ7KnEL9/cUOSKRYR6k6K3YG5Y5dZJe0RTN0aCIg9Rv+Eae -pKl4ieIE6UWa/rEu/xEV7cRklDOGhTDlhbebu8Fpdy9M3ZB/l20/+H7LG5LLT6Cm -7opPPPr+x0go4uKkKwYKYDxv6LvF/wmQqdMQY/X7nE+/n/8ywjlVtjL/ZMRQIAN6 -lupnxJJhTOhHYF89GPD4jDQ+gMIv9NL60mhiRO9+Rc5fG6qozHYNd5C2559P9W92 -tbWSK5nUxfFI0byFc9NNLv1fq7ZdY1dzuFg0P/Q4yjyw7qugdg1Sv7ufNACEm5Rl -Wlh4zh6ii7CXVnge0wo/grGUCskJTfJ/xH16cZFrkSEJP4m4URRkcgD6FXAv+VHH -jMCJNMxIADKCJwBCxnvYaf+8Ii/iX8GP3/GtGjg8nQJo7Zlt1hT5R1xWabnc/+r5 -vxjz7LveUp66bdzoxxMoBFuSYuhIkauCz+sLcG9HYBr/fhtkzlptkWp2Zparn+Iq -Caxtf4A2r8wb2sAbzZycRdsXaQ5aG6pye4uUA69YJka4RYQ1vBEGQ1APmcg6sEHY diff --git a/dev/travis_secret_gpg_key.pgp.enc b/dev/travis_secret_gpg_key.pgp.enc deleted file mode 100644 index ecf0adb0..00000000 --- a/dev/travis_secret_gpg_key.pgp.enc +++ /dev/null @@ -1,18 +0,0 @@ -U2FsdGVkX1+sgU5WO8D5WNNm5U1j12+x1sN0eey2VcZS5MB6t9oTHUcQQVIwQfq5 -pJAd8us+905kSGTOMIS2HlvppZvDI/byuIsh+yPvoPHih8JH6rAYtgRUzfhUn9Ex -3OGSXvj3Hy9YigedJQphtGf/Zg+EAo+uNzHYTvqeEV7GF+ZlGrk/BJmd5+1Do07a -jNK5h5mqLgJ8xL6KoUwQliW1/JWgvfVVEVVaH2tW+QGEvU4tPRudHzT7Adlqc1TP -w1XK+kCMZLrL3smrQL66hWreMEsRkhxZ+3rzFU7pjbmkhMZ84epNUDR/ChiV1Srq -yUxFesZGXta+W+UDV3n7MB3JOBmOrCV1wes3XbpZ/SJbflisjQX6cuSrpu20cW/K -Jv3i1ygAixQjq5Ojno1TihBM38gGvliEIurTqkd3sYj3vgPwXu3KL6KA3IrQjyfp -qMgKYA2DWy3Wau04Ytc4UxP8H4eNV8DdvmCDsIzZA28XGjAiC4DJ2CHCKw0wk3U6 -MLj5zzSxHbXZ02xWRQDXuep7Cy+fzphg5Oh2sjj8MODR9BJGLo+RqSeXPy780nKR -hxiO60FADUNUuXPbabo9EGw7+TPsoMWG4oDoCgGKQGCTyuOxo3CcW64v6mF8skql -CsgXgu/PtySBwtDTOimEszhr2cy74C3YdpTSbTmDOytk1MO9GOgHz0aR1D/j15lQ -T2OllCXDJWzSeD5Fus3WtGuOm8nT9kE3qYxnLjPAPUEq5asyWTpQfJltMyeWUh8U -Inb5Fpj169ZtSPBm0l1L9Kkm9zzlAVBowXD25sekTleqTa/3FOO1GzPkmz84w46S -aCTsnPvY6J68dXR8epCT0Yeop6q9x3QLm/q2rUKrMXQYv3DYGpVyfVltoYcSgk/E -2TjzRRiaDccjbqW0FTd2x/70aoLdD8enJqcYpeOZghqpEcdzBBWDnmGD0N60obmx -9a9++3FmcmeCXv5BZUkDu8+Ljy+nZIn+lFgkr64Z3BDoBmfZ2YJciG33H/z9deCZ -JFL4N/+EDMC/7N0NgiYBPyfWAgmkjo8o7rphCPajYSD5DjRX6DVyyMAWOAE5DcsN -6BQv/Q0GhThcnOcgHvvp2QW0QTmssAKeGhcY6FGHzqqrkBPViohfdWJjWjgbrqkD From 34f478cfe321083f0ebd2e3157dbdbc6fa23fd5b Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 21:52:56 -0400 Subject: [PATCH 22/31] Update secrets --- .github/workflows/setup_action_secrets.md | 217 ---------------------- .github/workflows/setup_secrets.sh | 69 ------- .github/workflows/tests.yml | 109 +++++++---- dev/ci_public_gpg_key.pgp.enc | 70 +++---- dev/ci_secret_gpg_subkeys.pgp.enc | 67 +++---- dev/gpg_owner_trust | 9 + dev/gpg_owner_trust.enc | 20 +- dev/public_gpg_key | 2 +- dev/secrets_configuration.sh | 2 + dev/setup_secrets.sh | 211 +++++++++++++++++++++ 10 files changed, 370 insertions(+), 406 deletions(-) delete mode 100644 .github/workflows/setup_action_secrets.md delete mode 100644 .github/workflows/setup_secrets.sh create mode 100644 dev/gpg_owner_trust create mode 100644 dev/secrets_configuration.sh create mode 100644 dev/setup_secrets.sh diff --git a/.github/workflows/setup_action_secrets.md b/.github/workflows/setup_action_secrets.md deleted file mode 100644 index b0167627..00000000 --- a/.github/workflows/setup_action_secrets.md +++ /dev/null @@ -1,217 +0,0 @@ -========================== -GITHUB ACTION INSTRUCTIONS -========================== - -This file is a reference script for setting up secrets for github actions -(because currently we can't add heredocs to github yaml files like we can with -other CI tools) - -This file was designed to be used as a template. You can adapt it to -new projects with a few simple changes. Namely perform the following -search and replaces. - - -# TODO: re-setup template - - -```bash -cat .github/workflows/setup_action_secrets.md | \ - sed 's|GITHUB_USER|Erotemic|g' | \ - sed 's|PYPKG|xdoctest|g' | \ - sed 's|GPG_ID|travis-ci-Erotemic|g' | \ - sed 's|PKG_CI_SECRET|EROTEMIC_CI_SECRET|g' \ -> /tmp/repl - -# Check the diff -colordiff .github/workflows/setup_action_secrets.md /tmp/repl - -# overwrite if you like the diff -cp /tmp/repl .github/workflows/setup_action_secrets.md -``` - -To use this script you need the following configurations on your CI account. - -NOTES ------ - -* This script will require maintenance for new releases of Python - - -CI SECRETS ----------- - -Almost all of the stages in this pipeline can be performed on a local machine -(making it much easier to debug) as well as the CI machine. However, there are -a handful of required environment variables which will contain sensitive -information. These variables are - -* `TWINE_USERNAME` - this is your pypi username - twine info is only needed if you want to automatically publish to pypi - -* `TWINE_PASSWORD` - this is your pypi password - -* `EROTEMIC_CI_SECRET` - We will use this as a secret key to encrypt/decrypt gpg secrets - This is only needed if you want to automatically sign published - wheels with a gpg key. - -* `PERSONAL_GITHUB_PUSH_TOKEN` - - This is only needed if you want to automatically git-tag release branches. - - To make a API token go to: - https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token - -Instructions: - - Browse to: - https://github.com/Erotemic/xdoctest/settings/secrets/actions - - Do whatever you need to locally access the values of these variables - - echo $TWINE_USERNAME - echo $PERSONAL_GITHUB_PUSH_TOKEN - echo $EROTEMIC_CI_SECRET - echo $TWINE_PASSWORD - - For each one, click "Add Environment Variable" and enter the name - and value. Unfortunately this is a manual process. - -WARNING: - -Ensure that your project settings do not allow Forks to view environment -variables. - -TODO: Can you protect branches on GithubActions? Is that the default? - -TODO: Look into secrethub - -WARNING: If an untrusted actor gains the ability to write to a -protected branch, then they will be able to exfiltrate your secrets. - -WARNING: These variables contain secret information. Ensure that these -the protected and masked settings are enabled when you create them. - - -ENCRYPTING GPG SECRETS ----------------------- - -The following script demonstrates how to securely encrypt a secret GPG key. It -is assumed that you have a file `secret_loader.sh` that looks like this - -```bash - source secretfile -``` - -and then a secret file that looks like this - -```bash - #!/bin/bash - echo /some/secret/file - - export TWINE_USERNAME= - export TWINE_PASSWORD= - export EROTEMIC_CI_SECRET="" - export PERSONAL_GITHUB_PUSH_TOKEN="git-push-token:" -``` - -You might also want to make a `secret_unloader.sh` that points to a script that -unloads these secret variables from the environment. - -Given this file-structure setup, you can then run the following -commands verbatim. Alternatively just populate the environment -variables and run line-by-line without creating the secret -loader/unloader scripts. - -```bash -# THIS IS NOT EXECUTE ON THE CI, THIS IS FOR DEVELOPER REFERENCE -# ON HOW THE ENCRYPTED GPG KEYS ARE SETUP. - -# Load or generate secrets -source $(secret_loader.sh) -echo $EROTEMIC_CI_SECRET -echo $TWINE_USERNAME - -# ADD RELEVANT VARIABLES TO CIRCLECI SECRET VARIABLES -# https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables -# See previous CIRCLE_CI section for more details - -# HOW TO ENCRYPT YOUR SECRET GPG KEY -IDENTIFIER="Erotemic-CI " -GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') -echo "GPG_KEYID = $GPG_KEYID" - -# Export plaintext gpg public keys, private keys, and trust info -mkdir -p dev -gpg --armor --export-secret-subkeys $GPG_KEYID > dev/ci_secret_gpg_subkeys.pgp -gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp -gpg --export-ownertrust > dev/gpg_owner_trust - -# Encrypt gpg keys and trust with CI secret -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc -echo $GPG_KEYID > dev/public_gpg_key - -# Test decrpyt -cat dev/public_gpg_key -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc -GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc - -source $(secret_unloader.sh) - -# Look at what we did, clean up, and add it to git -ls dev/*.enc -rm dev/gpg_owner_trust dev/*.pgp -git status -git add dev/*.enc -git add dev/public_gpg_key -``` - - -Test Github Push Token ----------------------- - -The following script tests if your `PERSONAL_GITHUB_PUSH_TOKEN` environment variable is correctly setup. - -```bash -docker run -it ubuntu -apt update -y && apt install git -y -git clone https://github.com/Erotemic/xdoctest.git -cd xdoctest -# do sed twice to handle the case of https clone with and without a read token -git config user.email "ci@circleci.com" -git config user.name "CircleCI-User" -URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g') -echo "URL_HOST = $URL_HOST" -git tag "test-tag4" -git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" - -# Cleanup after you verify the tags shows up on the remote -git push --delete origin test-tag4 -git tag --delete test-tag4 -``` - - - -Github Action Local Test ------------------------- - - -```bash - # How to run locally - # https://packaging.python.org/guides/using-testpypi/ - cd $HOME/code - git clone https://github.com/nektos/act.git $HOME/code/act - cd $HOME/code/act - chmod +x install.sh - ./install.sh -b $HOME/.local/opt/act - cd $HOME/code/xdoctest - - load_secrets - unset GITHUB_TOKEN - $HOME/.local/opt/act/act \ - --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \ - --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \ - --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \ - --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \ - --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD diff --git a/.github/workflows/setup_secrets.sh b/.github/workflows/setup_secrets.sh deleted file mode 100644 index 118b1e8b..00000000 --- a/.github/workflows/setup_secrets.sh +++ /dev/null @@ -1,69 +0,0 @@ -__doc__=" -Development script for updating secrets when they rotate -" - -cd $HOME/code/xdoctest - -# Load or generate secrets -source $(secret_loader.sh) -echo $PYUTILS_TWINE_USERNAME -CI_SECRET=$EROTEMIC_CI_SECRET -echo "CI_SECRET = $CI_SECRET" - -# ADD RELEVANT VARIABLES TO THE CI SECRET VARIABLES - -# HOW TO ENCRYPT YOUR SECRET GPG KEY -# You need to have a known public gpg key for this to make any sense -IDENTIFIER="=Erotemic-CI " -GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') -echo "GPG_KEYID = $GPG_KEYID" - -# Export plaintext gpg public keys, private keys, and trust info -mkdir -p dev -gpg --armor --export-options export-backup --export-secret-subkeys ${GPG_KEYID} > dev/ci_secret_gpg_subkeys.pgp -gpg --armor --export ${GPG_KEYID} > dev/ci_public_gpg_key.pgp - -# Encrypt gpg keys and trust with CI secret -GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc -GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc -echo $GPG_KEYID > dev/public_gpg_key - -# Test decrpyt -cat dev/public_gpg_key -GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose -GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --list-packets --verbose - -source $(secret_unloader.sh) - -# Look at what we did, clean up, and add it to git -ls dev/*.enc -rm dev/*.pgp -git status -git add dev/*.enc -git add dev/public_gpg_key - - -_test_gnu(){ - - export GNUPGHOME=$HOME/tmp-gpg-testbed3 - mkdir -p $GNUPGHOME - ls -al $GNUPGHOME - chmod 700 -R $GNUPGHOME - - gpg -k - source $(secret_loader.sh) - CI_SECRET=$EROTEMIC_CI_SECRET - echo "CI_SECRET = $CI_SECRET" - - cat dev/public_gpg_key - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc - - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import - - gpg -k - # | gpg --import - # | gpg --list-packets --verbose - -} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 17f69e38..bfbdf8d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -231,6 +231,7 @@ jobs: path: ./wheelhouse/xdoctest*.whl deploy: + # Publish on the real PyPI name: Uploading to PyPi runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') @@ -243,33 +244,30 @@ jobs: uses: actions/download-artifact@v2 with: name: wheels - path: wheelhouse + path: dist - name: Show files to upload shell: bash - run: ls -la wheelhouse + run: ls -la dist + + # Note: + # See ../../dev/setup_secrets.sh for details on how secrets are deployed securely - name: Sign and Publish env: - # Secrets should be uploaded here: - # https://github.com/Erotemic/xdoctest/settings/secrets/actions - # Toggle comments to publish to the test pypi instead of the real one - #TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ - #EROTEMIC_TWINE_USERNAME: ${{ secrets.EROTEMIC_TWINE_USERNAME }} - #EROTEMIC_TWINE_PASSWORD: ${{ secrets.EROTEMIC_TWINE_PASSWORD }} - TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ - EROTEMIC_TEST_TWINE_USERNAME: ${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} - EROTEMIC_TEST_TWINE_PASSWORD: ${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} + TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} run: | ls -al GPG_EXECUTABLE=gpg $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys - export EROTEMIC_CI_SECRET=${{ secrets.EROTEMIC_CI_SECRET }} GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") VERSION=$(python -c "import setup; print(setup.VERSION)") @@ -278,30 +276,73 @@ jobs: pip install requests[security] twine --user GPG_KEYID=$(cat dev/public_gpg_key) echo "GPG_KEYID = '$GPG_KEYID'" - # - #export TWINE_REPOSITORY_URL=https://test.pypi.org/legacy/ - #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} - #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} - #MB_PYTHON_TAG=$MB_PYTHON_TAG \ - # DO_GPG=True GPG_KEYID=$GPG_KEYID \ - # TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ - # TWINE_USERNAME=$TEST_TWINE_USERNAME \ - # TWINE_PASSWORD=$TEST_TWINE_PASSWORD \ - # GPG_EXECUTABLE=$GPG_EXECUTABLE \ - # DO_UPLOAD=True \ - # DO_TAG=False ./publish.sh - # - export TWINE_REPOSITORY_URL=https://upload.pypi.org/legacy/ - export TWINE_USERNAME=${{ secrets.EROTEMIC_TWINE_USERNAME }} - export TWINE_PASSWORD=${{ secrets.EROTEMIC_TWINE_PASSWORD }} - MB_PYTHON_TAG="*" \ + MB_PYTHON_TAG=$MB_PYTHON_TAG \ DO_GPG=True GPG_KEYID=$GPG_KEYID \ TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ - TWINE_USERNAME=$TWINE_USERNAME \ TWINE_PASSWORD=$TWINE_PASSWORD \ + TWINE_USERNAME=$TWINE_USERNAME \ GPG_EXECUTABLE=$GPG_EXECUTABLE \ - DO_UPLOAD=False \ + DO_UPLOAD=True \ DO_TAG=False ./publish.sh + + test_deploy: + # Publish on the test PyPI + name: Uploading to Test PyPi + runs-on: ubuntu-latest + if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) + needs: [build_and_test_wheels, build_and_test_sdist] + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Download wheels and sdist + uses: actions/download-artifact@v2 + with: + name: wheels + path: dist + + - name: Show files to upload + shell: bash + run: ls -la dist + - name: Sign and Publish + env: + TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ + #TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} + #TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} + #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} + #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} + #PYUTILS_CI_SECRET: ${{ secrets.PYUTILS_CI_SECRET }} + TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} + TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} + EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} + run: | + ls -al + GPG_EXECUTABLE=gpg + $GPG_EXECUTABLE --version + openssl version + $GPG_EXECUTABLE --list-keys + export EROTEMIC_CI_SECRET=${{ secrets.EROTEMIC_CI_SECRET }} + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" + $GPG_EXECUTABLE --list-keys + MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") + VERSION=$(python -c "import setup; print(setup.VERSION)") + pip install twine + pip install six pyopenssl ndg-httpsclient pyasn1 -U --user + pip install requests[security] twine --user + GPG_KEYID=$(cat dev/public_gpg_key) + echo "GPG_KEYID = '$GPG_KEYID'" + MB_PYTHON_TAG=$MB_PYTHON_TAG \ + DO_GPG=True GPG_KEYID=$GPG_KEYID \ + TWINE_REPOSITORY_URL=${TEST_TWINE_REPOSITORY_URL} \ + TWINE_USERNAME=${TEST_TWINE_USERNAME} \ + TWINE_PASSWORD=${TEST_TWINE_PASSWORD} \ + GPG_EXECUTABLE=$GPG_EXECUTABLE \ + DO_UPLOAD=True \ + DO_TAG=False ./publish.sh + ### # Unfortunately we cant (yet) use the yaml docstring trick here diff --git a/dev/ci_public_gpg_key.pgp.enc b/dev/ci_public_gpg_key.pgp.enc index 40eb988e..0ac8eadd 100644 --- a/dev/ci_public_gpg_key.pgp.enc +++ b/dev/ci_public_gpg_key.pgp.enc @@ -1,35 +1,35 @@ -U2FsdGVkX18PbC4mZJwLbtMm65zwvM+KN4w7GV7m4kwm2+SS/hMmnUQzSDJAJCD5 -eA9Uix3Pw8/slqv63euDewhyuTminDNal3xHu+YhbG+nwoKfBYvRzL1Qr08mdc7L -0bjFVDMc6I42BlxWj8AOmS2sRPe6jgA3oiULKOYFnaF9bRYkFKiUTQEEwwXURiSr -BkXmRAYGcFmJZF5ylV3thSjjiGPvtuIek/qboZ29L7HY7OLPKC38f+59hoF/48yH -Wb2RhWpYakMuyMiziW44w5nQ66rFyT6Rcl5JtmUOZFNbHZd5NxeCzVgOqQwVSxNu -gTMjchrvQ40jwwA1lqO48+cSkzMA+7+MmqBDmVkpJdY9OBmrpkkipNFKn0zHqEYK -VmsBfiEUW6wuebaZMAwwsmmgGDa6d+JOFyzLx+za//OYuxOM6P3waBrnkuQyk632 -Q5NPjILq3ty9CL7CW94qK2/PW0fUJVUBgI61ljq0wfgGtoolL0WAD4DhCt7x3hdJ -bt03+CgdhI7QDnbt7XuBz68R5YyuP/2An9APkFswZXNyYjbqdQMJnjYzrVYAg2an -G+6Bc5o32B2IHfctii5nDiIPPb1ZQlha8SJl3JPYOpVT92rZ10jES+P5baBdm8tY -zShOsGJ8ecNzZQbqhb1oRW/J2cFrTfSiRRF0f9NBHWbARpO3+uF+pAXQluJMSZ86 -a1oP1LrnY/7YwqMR1AVi6zbpyefJ7jEAZNwLD1n36x5iAGJQ8ND56rQYbalBeh2o -ToY+Lmy/hAp1EiVVdQYTnoGM//zD/L29M77hL5Ft+mKUOhfBn3DlDbiMKwsgWT6G -MHu7oXynzl5CpuOEuN59Bx15yjycLMnDuBoYv8zeb8RsbwlvHqsR39mreKb4m1FE -iDIDkee5Vgxy/7ChCyDOtMcXqt7k40F4a/F/bNkya2OLHMHw6zULCBb3gPcvubS/ -Zw5man3QHMqKVjWm4MN5pR+8F0GytOR0bizIu+8r0vHZzZZE/PVzpWMf+8R9MOqe -cckxCmjmgth/4fmB8qn4fmnZc9AUWj1uOWROD0L5smNBpgfpyJgv3XMsPuBBlPP0 -Q4xntrcSmO57BTcbgP6VL3PN5sWllH3mJJQ2vyzypstFHH1Sx/GftokDDlZGUqzp -8PhM+Jb5UhrWRh0BV1CjNx2FCMXZnDrUFhkDxrFC1H1J6liXk4Oa8pxRjfOfhh+7 -BlrJmrrY5OStdqclgz3nkx2hvHE+mMrxA1IelBeqR3c31ZDgnpFR3/ifDSKAhPSr -Ub71GtmGEJDINbhFt8hvpJa5cUzszxY2s6aPKEBxtVGYI24UVNuPucHinChaJohy -yZ4m7Qh9AwHF4hqjEtaghTqRab9dJLcEuJ9961uHoigg6fTmr270bVXYQdPgSPHx -LHYz7gnKz8CONA1Le7viYEHZVJFV8fXQ820kS4rpfmICarh4u0Z2NLv23LYFzm2o -dJQcD0TZZKZunAwPJgDH4CRRUvqyAee2vZOEB5zVCH54OhizfPmwpfREgknstC2Y -5SKVCCQRX4uZmgDKnUZvPfICoyw0N9+uodEhSsTw3XI19jfbBzPlN3q29v1ZrYXh -FyuP3K59XACPSqSqyeUsipqoCtVA7V+tDb08JUzQfsvwoo9W8/q4EGe2TkxHQrR+ -tK2HOlNwj6dG8j2aw6vHKgq6cwQEWZ2tyBohnwoM1cDOWBrdore1tnazLPSpWWTX -qaWIr4u5PoDMSeTY6fdsK+xC10Y8FazRfcBehc4ZSbw8VqOrWQ5LJe7mEJJLmnRu -yv9KhwTVwhGO9DaX1OdyxpRscH6tiN7OSDGjg92c9wcTdzc8qozKocl/nHIowblW -RI0qex2UMKIhJZyMcvLxKmvax+bgplc8pMOdS5U+Pe2c5eUJmmBAXdwi6gQW8Slo -zbNMLwtQu6Ma6UiptikI1o6XLTL5i6zhWEHueEx0bdaor8vbfTCQe5a4bN/LKLs9 -T9/QtzSpS5bg7qgoZA+rFnKDLyRiaIY7YwNpb2JbdQpkhd00c/JasyZBl5R1xiZm -DngvTn1TjaAetwsunmRPH5ZE0SYM8VZ23sYQc9x0SJU6GjLgQeViU2pfv6j3KjSE -qkNRZ4yx28jObYPVVJhQweCH1RbdxpQvwSFq5AOhmYWzh6r49S/CAE581hfmi2x8 -QktYOrBGC/i18I/uzRIBCg== +U2FsdGVkX1+X4uezbbjDmxrEhCujtGYg5wZPLKLZd8hN1AfAI6kahNjAMQpBfehc +QzzWfdeE1gaeGUaigghQXjYOz4cNgC43ti0HiI6o5lCu9QgTDq5NOy8BZVyZvb6G +i57Z4OpwA2wU4i+QgUOA5aMifGEc0MeRzBMn0N6mAbxhgA3YJEyknatg3iTjlt5t +knmGNCu1fa2aDO/DvYtwRuIISDu7lCUFrw1Fi9riXGteoN4FZzju+bN2S4bQfjSe +WI156ZSTg9A5Os93u3Z6xofzchgUTnqAfNkxSLxbl8Jk5AEvZnP0YtLphcl9Hdgm +nKYs7v0sSSBdVeItFB2lOYc2iq5CiUxRPvXb4I3G47grEgBwqeKcvixb6dEs7X1N +/utcnTBFqjB+pa7LTH+scuNwqc7z5pJ1F0bY6fwmOwd8P8X2WTdDFDhiJx2lyrhu +zvWE1Ppu1iuQW8T9Bg8bLkyvtKjHCJTKSCVUHoEH+TpGHaUcI9M0XC4hJKIGeRVM +uTIUNhN3sOS0uZ6jjN17cyhj+9SAzkWgRiqoiHd/Hzs3XqPqTesgJCMUsQwSrgxz +pHCSz12rW4U0SdwHF7X8jFLMVYSD7VA73nPxEgwgGfBWuxOAb2oKj/iXkyY53nDB +SgWZ24d5i/SkGsz8cJRcu7zhhHki4rhiijG5dU/B7XLQGXpbiKmB/eWt2kdmdy6E +bcJNIqQWq3T02wV1rottXssf3OAL/kC99ZlPm3HcR//kiMQQ7sWR0lPJECA1Inyt +2nuTTxMd4LAq1lL/zVwAf7a0N1miHq3yCe1nW3elGUwwwdS4Q4YwrRIUrRNd+PmE +QTD7ki/pIJ0mi5N8wAwXiAsTUUtWC0eCOZLwft9lulrDZfWM1zJ/zrqLhIhUuX2W +xAAInK6UuJFzPb5+SKAzuNOBSOyM8GChro0Da9UKZgirp7GY8+0/mmKCCkAofZEZ +A8yvx+wZCzeNsW6+ZwH/CfGDcBW1wwtiEdTGeUpHsAoz6+GK9PvgbJFzHcmxJ5ng +GsDs1WpILXBo972sxteyEyyOFOHVxRvXFJsrA1o3O49DouHf80K8nKJcPDIFuC6L +kdFjEpvvRAUXTtJdjxJqpPh1aR4iiHQwkDyM+3WY36o/O+iRyXoJdltCZF96mBrj +K0yhVe+g4EulOhjAr+mKWxu9+n1TX2k6GUZrnh0Mux31ksNvoxqwhf9K+E1uCFDW +JKpOdqSlJBj+w2hTKlLp1VH6rw9aIh1YCmHBQSiPsQvqeXkvmbZtz7ZotlianO7K +f3lgBxNnvpGfokWZ8ZubUICo3rG7lwD1FZV2+AEQWya03noF1M7FVRCzb52DloSl +BeCnxxGe3PhZcSMqqrMltKETk+QcBLUMEoOqTKLMvihJ1VDk+rKAmEB55EV1qhli +DJrxaH0VSK3wzLKOkmmDeRRSyvMPtIpO8vuCpQQqjEVo+Du0BPfi/ItdXA/GijXu +RMv60zYzZUCXz2f/FVOsNVq+iugd4Z/6QA35P3LMizjDjiwSza/otNQSQCGUVdpc +7/uVGAY3v/nDN1xlP5HtVZUleRTKcvw/uu/077szAL6tXSLBRwo8zH4wZ3uavyAC +b9jg92zoBIMsHxITc03Vih2d+L4qEbWuyUaNxFW6vmbk22P7zl+8uGSWYu9PqdEO +/985GXfzS5eAMW/N3hgkVHYEjH7WdudhSI8FImBvrurlg9/qm/WaxqPGTg+MFU1v +bAp9uRV2XZuSlmeT4BKadYt7N5lDnmgueF5q3hErTPghABzdL9VFA+8KebJU6IoH +bJxZbkRP+QQCOOYGlyWFICxTxorESOZt4izqsXu+Bwryjt0FzFYpcvo2XIzEn4Yz +alw1I8Xcy0aVLdffNeAmtskeshitIBMAb44mH2UEmTZ/+ITIK/zFnpkXipg+6p8P +u0P0MExTALCDee0KcOC3czgIqtv/uQ8M4ymi8nZNEdOdDyjl4ugwA1QAY6OVQLnj +RnUn46jgotEFdF0aNMcXdylR1zuboDNpYBwuWNpzVn0k3zTaslB2QrJk4saoTJfh +eFuY1Y8gTDugDwtSUo+n+OyiZpfykUv5ASIfwjz4XufR2FCT3GNOxs9CbACSEZkJ +dptkzX6JeUhxXxP/zQYn3MisdWtjmpPKPBz1irwnKOeYUQYYUrD1YhoJo+JAUWVh +2t9jh8KHphkD0FNM7hjaAg== diff --git a/dev/ci_secret_gpg_subkeys.pgp.enc b/dev/ci_secret_gpg_subkeys.pgp.enc index e4cfaa73..3d37c446 100644 --- a/dev/ci_secret_gpg_subkeys.pgp.enc +++ b/dev/ci_secret_gpg_subkeys.pgp.enc @@ -1,40 +1,27 @@ -U2FsdGVkX19mtCufSuMCTKwPlW3lZByYR4s/O8BcJCjIwXL6Zp7wKhgIfsSGVIHp -wo9elAW34nB2j5T4qRlLMfMWxaDuvTcM1Z5jmm/5hlaShgi5X/XLdyCwHc6qR7/X -cvJmvXzKqAwWyXoDy7jyFxShvgs1Fk1q21UnEyQNqag5ik05krC8eUtmgSpUosK1 -PPObGs5xFKALylw/N0VgzmraMVCkApxAshhTuMosUmyIEJnY/Q8DdaXcF4624RHU -hAVD+7XiM0vm6avu8PGqDxYwbLoGeUAcTPouhLpNuUO3N7TCAyceUJoeEE4ulJdA -OLrtYTWjSTxzQq9o7TdRv2l6/UobMrhwn7Jk55/UTJJVMGheEojL6tx6abUt1oMy -8pFN8M6mKvPMNzE2yhAJl+PJNMTPc1HFYtapHExtIizwrxl/IptxPxoMqDHxGjYJ -3EwOrBA1du6eU8XxBrZRXWQMciECBqnidVHrUBnoiaN3Bt+t+1m75hla1L/djQxQ -qq7uckwnh2gphgS/iGOOxaeQ69vcHky4NnBQpoAB/U4PVQNQcGvsrW/DU4wcb3kx -jslz6X/ujfB0B53ANDXsHxin5EJ1p2ldLZunFyl2HVWLNhg7XQXvh1/fZKCJtCbq -LqsmgCSE0YKbDqtFS/qh+LXZCabzHuFMwrPSFIe5QIBSImgU+UblgOkBY03zB8nE -CN7X2KglovpkWdM3T/nU+r3Utgv/n5PStqCs9JVan5o2Op2idUGZUcuvbJ5AZR9A -5eTEbPg4jMeXzJAZKGqop1yL4Z7fD/3DpG2MDBRPTuB4WuDhR4wKvNubTX1V6dJQ -AItNjQ8VN9DtBpJz4cmtpGju0h1tHRLIVxvlBNfm68diFM7ZsArypIy6DnBraEAH -xnvktO716TuGb1Mv3pm1GwKKjXuXWHV1/6Qd/N6ehvSrkCmmlBM6WiGZYvnllj/X -XglaJ5CDgnG/OmXkQWFZk9eYTHnTeyITVTaT4zG95AVqjePRblkwwCy5fv5R3Y14 -w/zaGxfS4VUoZBIBRAsnjLybKpkhsz5BvXr0wLYoD/RyNk/6xRuxD+FSC8bx4pe4 -zWdJeGg4kcHremMEtSwcQClOBgTlHHWBNhIpi6GjhlMbaq5E2xwiaFzxdOqN0aDR -VqnvklQGaWWunIwLkY4WJyGuNH5lehffRIFscT0rsLtAi/t/PcU8Oh25+078Ed+/ -7eJM5JjIHMXNCEmxN60Y4eNRYfe2FQUOoxIfjv1IvmtEegIzj5Mdc4ZDzgNjLmYp -MqRotXtDBZ1pku0fnHwjiveDzzT5MzTX3GtV2Grx7t38YeT3fJR/r1P4dDJfdKGR -1ukq052lmRdiIvUJEE0ViB+uYu3mngywDpaggtJgcjiVqaytdAjcjBuD+0zMAckQ -vJ6TTVdr4chjEbQXPZLbAduQMtgTCwrJThf4ZUSIym/YNXky24CyLFRwWaLV/oIE -4ixuqWXq2DGO+GYEUECNEb0+IEX8/0l6mvfn9FUF9+4Z7zvdgi2m1/jIcpz/aK+J -0lQ72ZECDd+yXffxbGLOVtUmk9x7MT/6alR11C4ll7YfZAP/GBGvOqjpHa3gsVPP -dK4fXRi+LQH40ZEsJ6ByZazG40Pyhr8SAK/VoXMJ65jTDFkoIAgkU6AWAwebE7LM -ys8KiKIh3YvST1ZBe29/CDb72JHgT8fxzFkVRmb1E9vuHz3eTsgEZYEuqCxLyJGA -+EbudFiTJNlCXv/V+CkJAk8T1C5ZoYIAM27k7Hi8RT8oM6gU3FfPnCzhaw50+rsX -xpX/HBA1Hz346U9IPGWLlbRJeQZprMFeI78oTL4U5kXFb0+RgZEWVpyyViqb+ka4 -5DL5i+K5Vf5FFOs23FkqTY7e+8tFfne0le1pVBi9CHztKFlmQ+vM+Ctv9HJVVIGS -f/w30wv9T1o91yYSf1y1G0spZnT8L0RtSx7EKc5QHaSrK075NAhgO4dSAs1R4twv -tyXZ0l+ztJ4jzo6ldFsQIdjyb1oAC3bL8qDfAJYn/pC31+30kHth9ZMv5XxUAxin -/x4/VpIIwBdUfQVSxTNMMbLxMK7cbIKQKvYvqfeJC0KTVJ3CDC8dKAh4NBdApRIV -gk6O7tyuIU8n+FdH9EtEePcOmoTkiGBqr2zg84W1D8/8rbCeAAgwY4b1tiVhwrFd -czVp07eblU953fhkrJtwOfJDxZlPtzTv7DWI1jJbPjUWkv7lSQoSZdAuQLQfSobs -nOqL58LrVtMuuHk5f31zx3yGpKudyZdxLtcQ9Fth8gJ/cLjG15dX75tPciv3j2Cv -BLKjWA4Hw0nDrC6A3qEteMmFFvCxuawS0JxaBNQg9zRLwz9NPZ5mAku72NtM4r0A -bdt+SrjsnEyvRAK7TZikVHhOy/aiNO56bST0O1eJjBchRyKAtUPwpijgWpXIcWqB -xf9/IJfJMpm6wiKBeQeNQPtEJwuaS3P73cYAy2nycZbBjuI+mtcVRe8daIVMnPBy -bktPQNqYHPFPRBV96lRPd/FTAzQrW5sjPdnPM7mp86EC6/GXCOExaYP3a1aY8LXP +U2FsdGVkX18p5fLvjvY0CGnafpqx3WPqzfkIY8I/AIM4aCEjzz2DldkFUoYYwGLc +FB4E10U1cxIByCK9Nk2aSjhognaIP4IOWCwD7ti1L0zy0CuZdlzgWVdH26xs6EZ8 +TKJD1STS/oexppTzJFbHUC5MqR/wMSNYFK/OQbhEjidxsrRH2P4NSQiYq34s3vU4 +JJsjnQUREdDYgctcCempI0FkPC/aAYkkS14j+iBnvQ6gM/OnovhTP/4iPLhraTCq +qYFcP0cdQ9V1puEOnxSCA7qNXMynwE5HvSft/Q+/+1RhMgMlKaXrd3Hhcs+nxrxO +X8zlSI63b2Q7K4XIQ2RpTnirUVSrN//WE5TOSY9GC+XDm6VSJ6ncAf7qWB0Cvycs +o51DbzT8iGflRXYH0PtcLkyD/3dBU4WaZFT9HPmpWHT51oh7i6gjJxDaC8YO8GGd +2VMnFvXGH1goqIThLIkmqqjyWBCinrWsxpBOnh9sgpOUIV5GdrucJb9kOSC+jCt6 +E4nML+R3f8OZt747gAmA+LAFHAPCYt+pHzXN06XPan5lUe+tkMjeLWhJN6DE2hy/ +AFCjoXXNtvq0WzPFaqw19ryLMsvH6ocZz3tOIJgBOVWFLbkG0uwFZSWKF+zmedsT +OtYM9QS87fdoIt5FHHs0Ic4/v3SzBXPz1JJDbGanqv5VC8NTFKle7YqlQAmi2/Mx +OsYxUGa2Zw5hU7S4TZgod0j/Ip/fAUeG43o4Kh+br0ivCYf0RO/LgUbC0l9oEGco +MzE5vxyh++21RnxhlZ+vdcU+KY5EOCwor43TodM7RShYGF7+YwpeA8CpfC85VQWw +18KIWxfCed0SM5kDKpZ0ISNwqofL1H0tBPYP9X9it5v0U0zhB9m5Gr7eS2Hqu1dk +Z3q0Kz8CbflLx8DNQ6MqR5iD63G5EwGqPtb7RBgBRZ0vnG1yXl52yTocI4aO6Iyt +kq/DOvyp4JVc5xAvvajstt87eYyST0konklSE/uSTxYDRcSuOSeP3VdP0K1E5H/L +JshVis++7cULrEVFa6/00ojsmAr2eU9dcWMy/9U1J/PNtUAi6JhVCZy1cIm2F4z2 +f4pOuey/C4Fs2izRObQ+v4lgfG0nJiIuJeovYB3qZ5/+P4HKOkArMIUUdO3QG67f +T7Y2x4BmcalE7fVtNhb4LxCfdqu/ku7jpY1VdkBnTgjno4E+2OSYbySbkGSlF+XR +lFPJToa3lSHf734rzIRyKcfRfccycGvdy5vIiSmWa8XryFZPw/BSHncv2p9YyCvC +JNsykxc4HHxx/AoYA4hlqDETz7PwaBoEf00yr3psL3486UWZvrIAzLsv9Fw/pwM7 +o8vKt2U7QnLnp5HuuoBEpqi3bUazMyWCoV2TfDZ+3BQYlmtWgpHE4TzN0VD3R6qB +mwobN58NX79ZS/U8cFcPxWzRCdWqv7Zd8KWtNIfN8SD28KamuY3fUvSbFId5v7ZS +uI3KZZl7yos0LHuS1jWBRqUZPNEfoFr2xg2PHZy28DewECxURCZADd8CcWPQoC9u +L+1yKxS8oD1Y/hmCg1fNjyX2kS3ZDNATtZX3mnMxkMNF/wmhLBNdmbkT99hjdj8o +3y7dpftOIC/njCEYWFlIy/eF3F7J+UUOJ4b1zSs1w5lTnTje4uTZ9Z5dAnLO1Myu +JE7E57qVJdlVBwtFYSJtjQVoYqNZjEX4+aAEFv2klAW+8quHJI9AvC0ph3YUw8si diff --git a/dev/gpg_owner_trust b/dev/gpg_owner_trust new file mode 100644 index 00000000..d51405be --- /dev/null +++ b/dev/gpg_owner_trust @@ -0,0 +1,9 @@ +# List of assigned trustvalues, created Thu 23 Sep 2021 09:31:07 PM EDT +# (Use "gpg --import-ownertrust" to restore them) +98007794ED130347559354B1109AC852D297D757:6: +2B9F0ED4BB49F26164A8B45DD6DBFC409CB9B78B:6: +F7C1C97A4EF2CF9643D94C666CC05F155B12AB36:6: +A1198702FC3E0A09A9AE5B75D5A1D4F266DE8DDF:6: +A490D0F4D311A4153E2BB7CADBB802B258ACD84F:6: +4AC8B478335ED6ED667715F3622BE571405441B4:6: +4AED9D3F16DB488FD7F33926C6AA1A4F42B93ED9:6: diff --git a/dev/gpg_owner_trust.enc b/dev/gpg_owner_trust.enc index 7c8e6fcf..2c2ea1a4 100644 --- a/dev/gpg_owner_trust.enc +++ b/dev/gpg_owner_trust.enc @@ -1,10 +1,10 @@ -U2FsdGVkX19u3udL4wxqGFCI5wmjZgfGufyvYlKI54O8zYZDFEHwY6RCMBTBELSv -G8RB+sZNmzhCLsyhgAT1QRRnKpYhetf3fw0HB9tnr1SqHdwzpYlUw6yUVBQOpasx -WhOXhN/1lOWNZqfNY1T6WarXXXO3mpGjpT6G0Mrf5SneRaemAZH9TgqiwlthpWqF -IjpvlqyCS6YbtXWNTb+A1pm72dmi8zcqrD7WAro0sHF+WtC6xE+WOEmUB52u0RXN -PMGK5lzv22iy1aFRVb29GYOtQddGDwIDnGLwwkt72ft+0qjfZVokyziaJdAss4MU -YuyxAwS0DAhBfu6BjW8TtBB7uFEKHABm+HyEi4W72e1ZTbylv/H9TXXzofhbNAPS -2Pn4axMy50zeIZeZHwi6fdQbMVfz2vamWhA2et04H+oiGnSMSorXCgZAWqYboM+I -2aCbUc9/I+Rju08p7gfJaCLhmEkRqD0RwjpqX50xyXcIcHA+9zpohEwGcRmnkZfJ -LHvNScvuSJum8EE6pCn+3WBaXOGGgEbooWzbQ/fHgMGjqc9yiVrcPL4s3NNMQNtl -3D8+cYYmk53UgW7yRsWdyw== +U2FsdGVkX19vZXMNVobiorDi2gogjVIjZ1ExKRsQlJ2k1xGjqffueVAJ/cy7DHLr +wsKKW8BKRErz68v/+P1+GgAhG/GPQdQCbuW2fAX/BZ+bteBDN1hPGd+YCX/C69NU +RtrqtcFrZHpuhTBE1fulBuzeLNmRaMEcDTjxFpkZ6hy1fTOZaSzrjAH+2/xcBkm/ +xElyfTSsJmbhOjpPTFtj6JVoRI++t2okNNzVfFs1OtGiPglFxjChan+LaacCmggS +WnJtk0mOq4LKZeq5GQBVwgcIy15p4GxU1AofBcUIIkWOcwq++s1rFKJOgCASKUFQ +CGWFQ7/0CffqxNNmrKzMM3BJXKF2XP9bT49hlGqpQJpCD9o5IRB77U5Vo83xJOft +QUnTghHKGkZ7v0WRAFGme3OqtY9+rCKcGq9aNzR+qPkapwstj4pjAaoETSr6V3tX +ORU4xqyo5EGURS9zK1d2lW4t5Y9LKxX0mjWbCVtPwdAHEOfDuA0fpVnoxQZ5vBp8 +qEA3u5V/bjRN8KP/07SNADHZibs+s1DiStCvbdyexNiOBLuYEime0qpgrz180NNQ +m5m9EsxHaBsZSnJcKO3j1Q== diff --git a/dev/public_gpg_key b/dev/public_gpg_key index 91ee9e69..8b137891 100644 --- a/dev/public_gpg_key +++ b/dev/public_gpg_key @@ -1 +1 @@ -70858F4D01314BF21427676F3D568E6559A34380 + diff --git a/dev/secrets_configuration.sh b/dev/secrets_configuration.sh new file mode 100644 index 00000000..09933e9d --- /dev/null +++ b/dev/secrets_configuration.sh @@ -0,0 +1,2 @@ +export VARNAME_CI_SECRET="EROTEMIC_CI_SECRET" +export GPG_IDENTIFIER="=Erotemic-CI " diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh new file mode 100644 index 00000000..a7adaa5e --- /dev/null +++ b/dev/setup_secrets.sh @@ -0,0 +1,211 @@ +__doc__=' +============================ +SETUP CI SECRET INSTRUCTIONS +============================ + +TODO: These instructions are currently pieced together from old disparate +instances, and are not yet fully organized. + +The original template file should be: +~/misc/templates/PYPKG/dev/setup_secrets.sh + +Development script for updating secrets when they rotate + + +The intent of this script is to help setup secrets for whichever of the +following CI platforms is used: + +../.github/workflows/tests.yml +../.gitlab-ci.yml +../.circleci/config.yml + + +========================= +GITHUB ACTION INSTRUCTIONS +========================= + +* `PERSONAL_GITHUB_PUSH_TOKEN` - + This is only needed if you want to automatically git-tag release branches. + + To make a API token go to: + https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token + + +========================= +GITLAB ACTION INSTRUCTIONS +========================= + + ```bash + cat .setup_secrets.sh | \ + sed "s|utils||g" | \ + sed "s|PYPKG||g" | \ + sed "s|travis-ci-Erotemic||g" | \ + sed "s|CI_SECRET||g" | \ + sed "s|GITLAB_ORG_PUSH_TOKEN||g" | \ + sed "s|gitlab.org.com|gitlab.your-instance.com|g" | \ + tee /tmp/repl && colordiff .setup_secrets.sh /tmp/repl + ``` + + * Make sure you add Runners to your project + https://gitlab.org.com/utils/PYPKG/-/settings/ci_cd + in Runners-> Shared Runners + and Runners-> Available specific runners + + * Ensure that you are auto-cancel redundant pipelines. + Navigate to https://gitlab.kitware.com/utils/PYPKGS/-/settings/ci_cd and ensure "Auto-cancel redundant pipelines" is checked. + + More details are here https://docs.gitlab.com/ee/ci/pipelines/settings.html#auto-cancel-redundant-pipelines + + * TWINE_USERNAME - this is your pypi username + twine info is only needed if you want to automatically publish to pypi + + * TWINE_PASSWORD - this is your pypi password + + * CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets + This is only needed if you want to automatically sign published + wheels with a gpg key. + + * GITLAB_ORG_PUSH_TOKEN - + This is only needed if you want to automatically git-tag release branches. + + Create a new personal access token in User->Settings->Tokens, + You can name the token GITLAB_ORG_PUSH_TOKEN_VALUE + Give it api and write repository permissions + + SeeAlso: https://gitlab.org.com/profile/personal_access_tokens + + Take this variable and record its value somewhere safe. I put it in my secrets file as such: + + export GITLAB_ORG_PUSH_TOKEN_VALUE= + + I also create another variable with the prefix "git-push-token", which is necessary + + export GITLAB_ORG_PUSH_TOKEN=git-push-token:$GITLAB_ORG_PUSH_TOKEN_VALUE + + Then add this as a secret variable here: https://gitlab.org.com/groups/utils/-/settings/ci_cd + Note the value of GITLAB_ORG_PUSH_TOKEN will look something like: "{token-name}:{token-password}" + For instance it may look like this: "git-push-token:62zutpzqga6tvrhklkdjqm" + + References: + https://stackoverflow.com/questions/51465858/how-do-you-push-to-a-gitlab-repo-using-a-gitlab-ci-job + + # ADD RELEVANT VARIABLES TO GITLAB SECRET VARIABLES + # https://gitlab.kitware.com/computer-vision/kwcoco/-/settings/ci_cd + # Note that it is important to make sure that these variables are + # only decrpyted on protected branches by selecting the protected + # and masked option. Also make sure you have master and release + # branches protected. + # https://gitlab.kitware.com/computer-vision/kwcoco/-/settings/repository#js-protected-branches-settings + + +============================ +Relevant CI Secret Locations +============================ + +https://github.com/pyutils/line_profiler/settings/secrets/actions + +https://app.circleci.com/settings/project/github/pyutils/line_profiler/environment-variables?return-to=https%3A%2F%2Fapp.circleci.com%2Fpipelines%2Fgithub%2Fpyutils%2Fline_profiler +' + + +setup_package_environs(){ + __doc__=" + Setup environment variables specific for this project. + The remainder of this script should ideally be general to any repo. These + non-secret variables are written to disk and loaded by the script, such + that the specific repo only needs to modify that configuration file. + " + + echo ' + export VARNAME_CI_SECRET="EROTEMIC_CI_SECRET" + export GPG_IDENTIFIER="=Erotemic-CI " + ' | python -c "import sys; from textwrap import dedent; print(dedent(sys.stdin.read()).strip(chr(10)))" > dev/secrets_configuration.sh + + #echo ' + #export VARNAME_CI_SECRET="PYUTILS_CI_SECRET" + #export GPG_IDENTIFIER="=PyUtils-CI " + #' | python -c "import sys; from textwrap import dedent; print(dedent(sys.stdin.read()).strip(chr(10)))" > dev/secrets_configuration.sh +} + + +export_encrypted_code_signing_keys(){ + # You will need to rerun this whenever the signkeys expire and are renewed + + # Load or generate secrets + load_secrets + + source dev/secrets_configuration.sh + + CI_SECRET="${!VARNAME_CI_SECRET}" + echo "CI_SECRET=$CI_SECRET" + echo "GPG_IDENTIFIER=$GPG_IDENTIFIER" + + # ADD RELEVANT VARIABLES TO THE CI SECRET VARIABLES + + # HOW TO ENCRYPT YOUR SECRET GPG KEY + # You need to have a known public gpg key for this to make any sense + + MAIN_GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$GPG_IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') + GPG_SIGN_SUBKEY=$(gpg --list-keys --with-subkey-fingerprints "$GPG_IDENTIFIER" | grep "\[S\]" -A 1 | tail -n 1 | awk '{print $1}') + echo "MAIN_GPG_KEYID = $MAIN_GPG_KEYID" + echo "GPG_SIGN_SUBKEY = $GPG_SIGN_SUBKEY" + + # Only export the signing secret subkey + # Export plaintext gpg public keys, private sign key, and trust info + mkdir -p dev + gpg --armor --export-options export-backup --export-secret-subkeys "${GPG_SIGN_SUBKEY}!" > dev/ci_secret_gpg_subkeys.pgp + gpg --armor --export ${GPG_SIGN_SUBKEY} > dev/ci_public_gpg_key.pgp + gpg --export-ownertrust > dev/gpg_owner_trust + + # Encrypt gpg keys and trust with CI secret + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc + echo $GPG_KEYID > dev/public_gpg_key + + # Test decrpyt + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --list-packets --verbose + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc + cat dev/public_gpg_key + + unload_secrets + + # Look at what we did, clean up, and add it to git + ls dev/*.enc + rm dev/*.pgp + rm dev/gpg_owner_trust + git status + git add dev/*.enc + git add dev/gpg_owner_trust + git add dev/public_gpg_key +} + + +_test_gnu(){ + export GNUPGHOME=$(mktemp -d -t) + ls -al $GNUPGHOME + chmod 700 -R $GNUPGHOME + + source dev/secrets_configuration.sh + + gpg -k + + load_secrets + CI_SECRET="${!VARNAME_CI_SECRET}" + echo "CI_SECRET = $CI_SECRET" + + cat dev/public_gpg_key + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc + + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | gpg --import-ownertrust + + + gpg -k + # | gpg --import + # | gpg --list-packets --verbose +} From 2aa2488ad2b1f5f2ac120f17532a09f9ba1b36be Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 22:18:07 -0400 Subject: [PATCH 23/31] wip --- .github/workflows/tests.yml | 24 +++++++++++++++++------- dev/setup_secrets.sh | 4 ++-- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bfbdf8d4..e61cb84d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,11 +88,12 @@ jobs: #needs: [lint] strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] + #os: [ubuntu-latest, windows-latest, macOS-latest] + os: [ubuntu-latest] python-version: - - '3.5' - - '3.6' - - '3.7' + #- '3.5' + #- '3.6' + #- '3.7' - '3.8' arch: [auto] #cibw_build: [cp3*-*] @@ -264,9 +265,14 @@ jobs: $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys + echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || true GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import-ownertrust + $GPG_EXECUTABLE --list-keys || true GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || true + echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") @@ -289,7 +295,7 @@ jobs: # Publish on the test PyPI name: Uploading to Test PyPi runs-on: ubuntu-latest - if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) + #if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) needs: [build_and_test_wheels, build_and_test_sdist] steps: - name: Checkout source @@ -320,11 +326,15 @@ jobs: GPG_EXECUTABLE=gpg $GPG_EXECUTABLE --version openssl version - $GPG_EXECUTABLE --list-keys - export EROTEMIC_CI_SECRET=${{ secrets.EROTEMIC_CI_SECRET }} + $GPG_EXECUTABLE --list-keys || true + echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || true GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + $GPG_EXECUTABLE --list-keys || true GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import + $GPG_EXECUTABLE --list-keys || true + echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh index a7adaa5e..9483469c 100644 --- a/dev/setup_secrets.sh +++ b/dev/setup_secrets.sh @@ -197,12 +197,12 @@ _test_gnu(){ cat dev/public_gpg_key GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import - GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | gpg --import-ownertrust + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import gpg -k From 0dd986a013b531c2e1ae7c3b29b94064c8299b76 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 22:30:17 -0400 Subject: [PATCH 24/31] Note about new sign keys' s --- .github/workflows/tests.yml | 7 +------ CHANGELOG.md | 2 ++ dev/public_gpg_key | 2 +- dev/setup_secrets.sh | 2 +- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e61cb84d..9a2cfb61 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -302,7 +302,7 @@ jobs: uses: actions/checkout@v2 - name: Download wheels and sdist - uses: actions/download-artifact@v2 + usesGPG_KEYID: actions/download-artifact@v2 with: name: wheels path: dist @@ -313,11 +313,6 @@ jobs: - name: Sign and Publish env: TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ - #TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} - #TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} - #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} - #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} - #PYUTILS_CI_SECRET: ${{ secrets.PYUTILS_CI_SECRET }} TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} diff --git a/CHANGELOG.md b/CHANGELOG.md index bea2947a..8af49bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm * Added GitHub actions to the CI * Disabled workaround 16806 in Python 3.8+ +* New CI GPG Keys: Erotemic-CI: 70858F4D01314BF21427676F3D568E6559A34380 for + reference the old signing key was 98007794ED130347559354B1109AC852D297D757. ### Fixed diff --git a/dev/public_gpg_key b/dev/public_gpg_key index 8b137891..91ee9e69 100644 --- a/dev/public_gpg_key +++ b/dev/public_gpg_key @@ -1 +1 @@ - +70858F4D01314BF21427676F3D568E6559A34380 diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh index 9483469c..ddda23a7 100644 --- a/dev/setup_secrets.sh +++ b/dev/setup_secrets.sh @@ -161,7 +161,7 @@ export_encrypted_code_signing_keys(){ GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc - echo $GPG_KEYID > dev/public_gpg_key + echo $MAIN_GPG_KEYID > dev/public_gpg_key # Test decrpyt GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose From 1b7ba73e9cfa4ae58a7fc938f81a070a0c334ebc Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 22:32:10 -0400 Subject: [PATCH 25/31] wip --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9a2cfb61..8b010774 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -302,7 +302,7 @@ jobs: uses: actions/checkout@v2 - name: Download wheels and sdist - usesGPG_KEYID: actions/download-artifact@v2 + uses: actions/download-artifact@v2 with: name: wheels path: dist From 078588fb61ca838fa269cedabd8fb4bdc1e4f46a Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 22:54:06 -0400 Subject: [PATCH 26/31] Reenable all tests --- .github/workflows/tests.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b010774..9e6a1788 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -265,13 +265,9 @@ jobs: $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys - echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || true - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import-ownertrust - $GPG_EXECUTABLE --list-keys || true - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || true + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys @@ -322,13 +318,9 @@ jobs: $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys || true - echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || true - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - $GPG_EXECUTABLE --list-keys || true - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || true + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import-ownertrust echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys From 2d7311914226347cdf34402f9a2b900845ca03e2 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 23:04:11 -0400 Subject: [PATCH 27/31] wip --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9e6a1788..a8acc94e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -144,7 +144,6 @@ jobs: with: platforms: all - # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml #- name: Build wheels # uses: pypa/cibuildwheel@v1.11.0 @@ -266,8 +265,8 @@ jobs: openssl version $GPG_EXECUTABLE --list-keys GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys @@ -292,6 +291,7 @@ jobs: name: Uploading to Test PyPi runs-on: ubuntu-latest #if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) + if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags') needs: [build_and_test_wheels, build_and_test_sdist] steps: - name: Checkout source @@ -319,8 +319,8 @@ jobs: openssl version $GPG_EXECUTABLE --list-keys || true GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_gpg_owner_trust.enc | $GPG_EXECUTABLE --import - GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import echo "Finish Decrypt Keys" $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" $GPG_EXECUTABLE --list-keys From ef888277285ee6d854f7fd61e9a37a0f38fce818 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 23:05:31 -0400 Subject: [PATCH 28/31] wip --- .github/workflows/tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a8acc94e..e52e31ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,12 +88,11 @@ jobs: #needs: [lint] strategy: matrix: - #os: [ubuntu-latest, windows-latest, macOS-latest] - os: [ubuntu-latest] + os: [ubuntu-latest, windows-latest, macOS-latest] python-version: - #- '3.5' - #- '3.6' - #- '3.7' + - '3.5' + - '3.6' + - '3.7' - '3.8' arch: [auto] #cibw_build: [cp3*-*] From e30cc850932080e93cd47715a0de312bd9e66536 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 23:12:25 -0400 Subject: [PATCH 29/31] wip --- publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish.sh b/publish.sh index 1c9c912e..8dc4070b 100755 --- a/publish.sh +++ b/publish.sh @@ -305,9 +305,9 @@ if [[ "$DO_UPLOAD" == "True" ]]; then for WHEEL_PATH in "${WHEEL_PATHS[@]}" do if [ "$DO_GPG" == "True" ]; then - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --skip-existing --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } else - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --skip-existing --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } fi done echo """ From 5a2cd433acab88d2320d4466426c05b68d54b20f Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 23:17:27 -0400 Subject: [PATCH 30/31] wip --- .github/workflows/tests.yml | 2 ++ publish.sh | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e52e31ba..e5c365c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -263,6 +263,7 @@ jobs: $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys + echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import @@ -317,6 +318,7 @@ jobs: $GPG_EXECUTABLE --version openssl version $GPG_EXECUTABLE --list-keys || true + echo "Decrypting Keys" GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import diff --git a/publish.sh b/publish.sh index 8dc4070b..96eb6901 100755 --- a/publish.sh +++ b/publish.sh @@ -34,6 +34,7 @@ Usage: # Set your variables or load your secrets export TWINE_USERNAME= export TWINE_PASSWORD= + TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" source $(secret_loader.sh) @@ -107,6 +108,13 @@ DO_TAG=$(normalize_boolean "$DO_TAG") TWINE_USERNAME=${TWINE_USERNAME:=""} TWINE_PASSWORD=${TWINE_PASSWORD:=""} +if [[ "$(cat .git/HEAD)" != "ref: refs/heads/release" ]]; then + # If we are not on release, then default to the test pypi upload repo + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://test.pypi.org/legacy/"} +else + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://upload.pypi.org/legacy/"} +fi + if [[ "$(which gpg2)" != "" ]]; then GPG_EXECUTABLE=${GPG_EXECUTABLE:=gpg2} else @@ -121,6 +129,7 @@ echo " === PYPI BUILDING SCRIPT == VERSION='$VERSION' TWINE_USERNAME='$TWINE_USERNAME' +TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL GPG_KEYID = '$GPG_KEYID' MB_PYTHON_TAG = '$MB_PYTHON_TAG' @@ -305,9 +314,13 @@ if [[ "$DO_UPLOAD" == "True" ]]; then for WHEEL_PATH in "${WHEEL_PATHS[@]}" do if [ "$DO_GPG" == "True" ]; then - twine upload --skip-existing --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD \ + --repository-url $TWINE_REPOSITORY_URL \ + --sign $WHEEL_PATH.asc $WHEEL_PATH --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; } else - twine upload --skip-existing --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD \ + --repository-url $TWINE_REPOSITORY_URL \ + $WHEEL_PATH --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; } fi done echo """ From 135e37976dd31dee4af443606203e1481797d58c Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 23 Sep 2021 23:57:31 -0400 Subject: [PATCH 31/31] Remove Deploy and Sign from CircleCI --- .circleci/config.yml | 303 +------------------------------------------ 1 file changed, 1 insertion(+), 302 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 75be81c1..f3eb92cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,191 +19,7 @@ __doc__: &__doc__ - | __doc__=""" - - ============ - CIRCLECI INSTRUCTIONS - ============ - - This file was designed to be used as a template. You can adapt it to - new projects with a few simple changes. Namely perform the following - search and replaces. - - ```bash - cat .circleci/config.yml | \ - sed 's|GITHUB_USER|Erotemic|g' | \ - sed 's|PYPKG|xdoctest|g' | \ - sed 's|GPG_ID|travis-ci-Erotemic|g' | \ - sed 's|CIRCLE_CI_SECRET|CIRCLE_CI_SECRET|g' | \ - tee /tmp/repl && colordiff .circleci/config.yml /tmp/repl - # overwrite if you like the diff - cp /tmp/repl .circleci/config.yml - ``` - - To use this script you need the following configurations on your - CircleCI account. - - NOTES - ----- - - * This script will require matainence for new releases of Python - - - CIRCLECI SECRETS # NOQA - -------------- - - - Almost all of the stages in this pipeline can be performed on a local - machine (making it much easier to debug) as well as the circleci-ci - machine. However, there are a handeful of required environment - variables which will contain sensitive information. These variables are - - * TWINE_USERNAME - this is your pypi username - twine info is only needed if you want to automatically publish to pypi - - * TWINE_PASSWORD - this is your pypi password - - * CIRCLE_CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets - This is only needed if you want to automatically sign published - wheels with a gpg key. - - * PERSONAL_GITHUB_PUSH_TOKEN - - This is only needed if you want to automatically git-tag release branches. - - To make a API token go to: - https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token - - Instructions: - - Browse to: - https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables - - Do whatever you need to locally access the values of these variables - - echo $TWINE_USERNAME - echo $PERSONAL_GITHUB_PUSH_TOKEN - echo $CIRCLE_CI_SECRET - echo $TWINE_PASSWORD - - For each one, click "Add Environment Variable" and enter the name - and value. Unfortunately this is a manual process. - - WARNING: Ensure that your CircleCI project settings do not allow Forks - to view environment variables. - - TODO: Can you protect branches on CircleCI, is that the default? - - - ANS: CircleCI provides contexts: https://circleci.com/docs/2.0/contexts/ - - TODO: Look into secrethub - - WARNING: If an untrusted actor gains the ability to write to a - protected branch, then they will be able to exfiltrate your secrets. - - WARNING: These variables contain secret information. Ensure that these - the protected and masked settings are enabled when you create them. - - - ENCRYPTING GPG SECRETS - ---------------------- - - The following script demonstrates how to securely encrypt a secret GPG key. It is assumed that you have - a file secret_loader.sh that looks like this - - ```bash - source secretfile - ``` - - and then a secretfile that looks like this - - ```bash - #!/bin/bash - echo /some/secret/file - - export TWINE_USERNAME= - export TWINE_PASSWORD= - export CIRCLE_CI_SECRET="" - export PERSONAL_GITHUB_PUSH_TOKEN='git-push-token:' - ``` - - You should also make a secret_unloader.sh that points to a script that - unloads these secret variables from the environment. - - Given this file-structure setup, you can then run the following - commands verbatim. Alternatively just populate the environment - variables and run line-by-line without creating the secret - loader/unloader scripts. - - ```bash - # THIS IS NOT EXECUTE ON THE CI, THIS IS FOR DEVELOPER REFERENCE - # ON HOW THE ENCRYPTED GPG KEYS ARE SETUP. - - # Load or generate secrets - source $(secret_loader.sh) - echo $CIRCLE_CI_SECRET - echo $TWINE_USERNAME - - # ADD RELEVANT VARIABLES TO CIRCLECI SECRET VARIABLES - # https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables - # See previous CIRCLE_CI section for more details - - # HOW TO ENCRYPT YOUR SECRET GPG KEY - IDENTIFIER="travis-ci-Erotemic" - GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}' | tail -c 9) - echo "GPG_KEYID = $GPG_KEYID" - - # Export plaintext gpg public keys, private keys, and trust info - mkdir -p dev - gpg --armor --export-secret-keys $GPG_KEYID > dev/ci_secret_gpg_key.pgp - gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp - gpg --export-ownertrust > dev/gpg_owner_trust - - # Encrypt gpg keys and trust with CI secret - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_key.pgp > dev/ci_secret_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc - echo $GPG_KEYID > dev/public_gpg_key - - # Test decrpyt - cat dev/public_gpg_key - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc - - source $(secret_unloader.sh) - - # Look at what we did, clean up, and add it to git - ls dev/*.enc - rm dev/gpg_owner_trust dev/*.pgp - git status - git add dev/*.enc - git add dev/public_gpg_key - ``` - - - TEST PERSONAL_GITHUB_PUSH_TOKEN - ------------------- - - The following script tests if your PERSONAL_GITHUB_PUSH_TOKEN environment variable is correctly setup. - - ```bash - docker run -it ubuntu - apt update -y && apt install git -y - git clone https://github.com/Erotemic/xdoctest.git - cd xdoctest - # do sed twice to handle the case of https clone with and without a read token - git config user.email "ci@circleci.com" - git config user.name "CircleCI-User" - URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g') - echo "URL_HOST = $URL_HOST" - git tag "test-tag4" - git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" - - # Cleanup after you verify the tags shows up on the remote - git push --delete origin test-tag4 - git tag --delete test-tag4 - ``` - - + Main CI has moved to github actions """ # " # hack for vim yml syntax highlighter version: 2 @@ -270,20 +86,6 @@ workflows: filters: <<: *__ignore_release__ - - gpgsign/cp38-38-linux: - filters: - branches: - only: - - master - - release - - - deploy/cp38-38-linux: - requires: - - gpgsign/cp38-38-linux - filters: - branches: - only: - - release jobs: @@ -380,85 +182,6 @@ jobs: path: .coverage destination: .coverage - .gpgsign_template: &gpgsign_template - <<: - - *common_template - steps: - - checkout - - run: - name: gpg_sign_dist - command: | - $PYTHON_EXE -m venv venv || virtualenv -v venv # first command is python3 || second is python2 - . venv/bin/activate - export GPG_EXECUTABLE=gpg - export GPG_KEYID=$(cat dev/public_gpg_key) - echo "GPG_KEYID = $GPG_KEYID" - echo "-- QUERY GPG VERSION..." - $GPG_EXECUTABLE --version - echo "-- QUERY OPENSSL VERSION..." - openssl version - echo "-- QUERY GPG KEYS (done twice as a workaround)" - $GPG_EXECUTABLE --list-keys || $GPG_EXECUTABLE --list-keys - echo "-- QUERY GPG KEYS (done twice as a workaround)" - $GPG_EXECUTABLE --list-keys || $GPG_EXECUTABLE --list-keys - echo "-- Decrypt and import GPG Keys / trust" - # note CIRCLE_CI_SECRET is a protected variables only available on master and release branch - echo "CIRCLE_CI_SECRET = $CIRCLE_CI_SECRET" - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" - $GPG_EXECUTABLE --list-keys - # The publish script only builds wheels and does gpg signing if DO_UPLOAD is no - pip install requests[security] twine wheel - echo "Execute the publish script in dry mode" - MB_PYTHON_TAG=$MB_PYTHON_TAG DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE CURRENT_BRANCH=release DEPLOY_BRANCH=release DO_UPLOAD=False DO_TAG=False ./publish.sh - - store_artifacts: - path: dist - destination: dist - - - .deploy_template: &deploy_template - <<: - - *common_template - steps: - - checkout - - run: - name: deploy - command: | - $PYTHON_EXE -m venv venv || virtualenv -v venv # first command is python3 || second is python2 - . venv/bin/activate - export GPG_EXECUTABLE=gpg - export GPG_KEYID=$(cat dev/public_gpg_key) - # Decrypt and import GPG Keys / trust - # note CIRCLE_CI_SECRET is a protected variables only available on master and release branch - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" - $GPG_EXECUTABLE --list-keys - # Install twine - pip install six pyopenssl ndg-httpsclient pyasn1 -U - pip install requests[security] twine wheel - # Execute the publish script for real this time - MB_PYTHON_TAG=$MB_PYTHON_TAG DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE CURRENT_BRANCH=release DEPLOY_BRANCH=release DO_UPLOAD=True DO_TAG=False ./publish.sh - # Have the server git-tag the release and push the tags - VERSION=$($PYTHON_EXE -c "import setup; print(setup.VERSION)") - # do sed twice to handle the case of https clone with and without a read token - URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g' | sed -e 's|git@||g' | sed -e 's|:|/|g') - echo "URL_HOST = $URL_HOST" - # A git config user name and email is required. Set if needed. - if [[ "$(git config user.email)" == "" ]]; then - git config user.email "ci@circleci.com" - git config user.name "CircleCI" - fi - if [ $(git tag -l "$VERSION") ]; then - echo "Tag already exists" - else - git tag $VERSION -m "tarball tag $VERSION" - git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" - fi - ################################### ### INHERIT FROM BASE TEMPLATES ### @@ -568,31 +291,9 @@ jobs: - PYTHON_EXE: pypy3 - # -- gpgsign + deploy - - gpgsign/cp38-38-linux: - <<: *gpgsign_template - docker: - - image: circleci/python:3.8 - - - deploy/cp38-38-linux: - <<: *deploy_template - docker: - - image: circleci/python:3.8 - - .__doc__: &__doc__ - | - - # Test GPG sign works - load_secrets - circleci local execute \ - -e CIRCLE_CI_SECRET=$CIRCLE_CI_SECRET \ - --job gpgsign/cp38-38-linux - - IMAGE_NAME=circleci/python:3.9 docker pull $IMAGE_NAME @@ -669,8 +370,6 @@ jobs: JOB_NAME=test_minimal/pypy3 circleci local execute --job $JOB_NAME - circleci local execute --job gpgsign/cp38-38-linux - JOB_NAME=test_full/pypy3 circleci local execute --job $JOB_NAME