diff --git a/.circleci/config.yml b/.circleci/config.yml index 75be81c1..f3eb92cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,191 +19,7 @@ __doc__: &__doc__ - | __doc__=""" - - ============ - CIRCLECI INSTRUCTIONS - ============ - - This file was designed to be used as a template. You can adapt it to - new projects with a few simple changes. Namely perform the following - search and replaces. - - ```bash - cat .circleci/config.yml | \ - sed 's|GITHUB_USER|Erotemic|g' | \ - sed 's|PYPKG|xdoctest|g' | \ - sed 's|GPG_ID|travis-ci-Erotemic|g' | \ - sed 's|CIRCLE_CI_SECRET|CIRCLE_CI_SECRET|g' | \ - tee /tmp/repl && colordiff .circleci/config.yml /tmp/repl - # overwrite if you like the diff - cp /tmp/repl .circleci/config.yml - ``` - - To use this script you need the following configurations on your - CircleCI account. - - NOTES - ----- - - * This script will require matainence for new releases of Python - - - CIRCLECI SECRETS # NOQA - -------------- - - - Almost all of the stages in this pipeline can be performed on a local - machine (making it much easier to debug) as well as the circleci-ci - machine. However, there are a handeful of required environment - variables which will contain sensitive information. These variables are - - * TWINE_USERNAME - this is your pypi username - twine info is only needed if you want to automatically publish to pypi - - * TWINE_PASSWORD - this is your pypi password - - * CIRCLE_CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets - This is only needed if you want to automatically sign published - wheels with a gpg key. - - * PERSONAL_GITHUB_PUSH_TOKEN - - This is only needed if you want to automatically git-tag release branches. - - To make a API token go to: - https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token - - Instructions: - - Browse to: - https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables - - Do whatever you need to locally access the values of these variables - - echo $TWINE_USERNAME - echo $PERSONAL_GITHUB_PUSH_TOKEN - echo $CIRCLE_CI_SECRET - echo $TWINE_PASSWORD - - For each one, click "Add Environment Variable" and enter the name - and value. Unfortunately this is a manual process. - - WARNING: Ensure that your CircleCI project settings do not allow Forks - to view environment variables. - - TODO: Can you protect branches on CircleCI, is that the default? - - - ANS: CircleCI provides contexts: https://circleci.com/docs/2.0/contexts/ - - TODO: Look into secrethub - - WARNING: If an untrusted actor gains the ability to write to a - protected branch, then they will be able to exfiltrate your secrets. - - WARNING: These variables contain secret information. Ensure that these - the protected and masked settings are enabled when you create them. - - - ENCRYPTING GPG SECRETS - ---------------------- - - The following script demonstrates how to securely encrypt a secret GPG key. It is assumed that you have - a file secret_loader.sh that looks like this - - ```bash - source secretfile - ``` - - and then a secretfile that looks like this - - ```bash - #!/bin/bash - echo /some/secret/file - - export TWINE_USERNAME= - export TWINE_PASSWORD= - export CIRCLE_CI_SECRET="" - export PERSONAL_GITHUB_PUSH_TOKEN='git-push-token:' - ``` - - You should also make a secret_unloader.sh that points to a script that - unloads these secret variables from the environment. - - Given this file-structure setup, you can then run the following - commands verbatim. Alternatively just populate the environment - variables and run line-by-line without creating the secret - loader/unloader scripts. - - ```bash - # THIS IS NOT EXECUTE ON THE CI, THIS IS FOR DEVELOPER REFERENCE - # ON HOW THE ENCRYPTED GPG KEYS ARE SETUP. - - # Load or generate secrets - source $(secret_loader.sh) - echo $CIRCLE_CI_SECRET - echo $TWINE_USERNAME - - # ADD RELEVANT VARIABLES TO CIRCLECI SECRET VARIABLES - # https://app.circleci.com/settings/project/github/Erotemic/xdoctest/environment-variables - # See previous CIRCLE_CI section for more details - - # HOW TO ENCRYPT YOUR SECRET GPG KEY - IDENTIFIER="travis-ci-Erotemic" - GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}' | tail -c 9) - echo "GPG_KEYID = $GPG_KEYID" - - # Export plaintext gpg public keys, private keys, and trust info - mkdir -p dev - gpg --armor --export-secret-keys $GPG_KEYID > dev/ci_secret_gpg_key.pgp - gpg --armor --export $GPG_KEYID > dev/ci_public_gpg_key.pgp - gpg --export-ownertrust > dev/gpg_owner_trust - - # Encrypt gpg keys and trust with CI secret - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_key.pgp > dev/ci_secret_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc - echo $GPG_KEYID > dev/public_gpg_key - - # Test decrpyt - cat dev/public_gpg_key - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc - - source $(secret_unloader.sh) - - # Look at what we did, clean up, and add it to git - ls dev/*.enc - rm dev/gpg_owner_trust dev/*.pgp - git status - git add dev/*.enc - git add dev/public_gpg_key - ``` - - - TEST PERSONAL_GITHUB_PUSH_TOKEN - ------------------- - - The following script tests if your PERSONAL_GITHUB_PUSH_TOKEN environment variable is correctly setup. - - ```bash - docker run -it ubuntu - apt update -y && apt install git -y - git clone https://github.com/Erotemic/xdoctest.git - cd xdoctest - # do sed twice to handle the case of https clone with and without a read token - git config user.email "ci@circleci.com" - git config user.name "CircleCI-User" - URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g') - echo "URL_HOST = $URL_HOST" - git tag "test-tag4" - git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" - - # Cleanup after you verify the tags shows up on the remote - git push --delete origin test-tag4 - git tag --delete test-tag4 - ``` - - + Main CI has moved to github actions """ # " # hack for vim yml syntax highlighter version: 2 @@ -270,20 +86,6 @@ workflows: filters: <<: *__ignore_release__ - - gpgsign/cp38-38-linux: - filters: - branches: - only: - - master - - release - - - deploy/cp38-38-linux: - requires: - - gpgsign/cp38-38-linux - filters: - branches: - only: - - release jobs: @@ -380,85 +182,6 @@ jobs: path: .coverage destination: .coverage - .gpgsign_template: &gpgsign_template - <<: - - *common_template - steps: - - checkout - - run: - name: gpg_sign_dist - command: | - $PYTHON_EXE -m venv venv || virtualenv -v venv # first command is python3 || second is python2 - . venv/bin/activate - export GPG_EXECUTABLE=gpg - export GPG_KEYID=$(cat dev/public_gpg_key) - echo "GPG_KEYID = $GPG_KEYID" - echo "-- QUERY GPG VERSION..." - $GPG_EXECUTABLE --version - echo "-- QUERY OPENSSL VERSION..." - openssl version - echo "-- QUERY GPG KEYS (done twice as a workaround)" - $GPG_EXECUTABLE --list-keys || $GPG_EXECUTABLE --list-keys - echo "-- QUERY GPG KEYS (done twice as a workaround)" - $GPG_EXECUTABLE --list-keys || $GPG_EXECUTABLE --list-keys - echo "-- Decrypt and import GPG Keys / trust" - # note CIRCLE_CI_SECRET is a protected variables only available on master and release branch - echo "CIRCLE_CI_SECRET = $CIRCLE_CI_SECRET" - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" - $GPG_EXECUTABLE --list-keys - # The publish script only builds wheels and does gpg signing if DO_UPLOAD is no - pip install requests[security] twine wheel - echo "Execute the publish script in dry mode" - MB_PYTHON_TAG=$MB_PYTHON_TAG DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE CURRENT_BRANCH=release DEPLOY_BRANCH=release DO_UPLOAD=False DO_TAG=False ./publish.sh - - store_artifacts: - path: dist - destination: dist - - - .deploy_template: &deploy_template - <<: - - *common_template - steps: - - checkout - - run: - name: deploy - command: | - $PYTHON_EXE -m venv venv || virtualenv -v venv # first command is python3 || second is python2 - . venv/bin/activate - export GPG_EXECUTABLE=gpg - export GPG_KEYID=$(cat dev/public_gpg_key) - # Decrypt and import GPG Keys / trust - # note CIRCLE_CI_SECRET is a protected variables only available on master and release branch - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust - GLKWS=$CIRCLE_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_key.pgp.enc | $GPG_EXECUTABLE --import - $GPG_EXECUTABLE --list-keys || echo "first one fails for some reason" - $GPG_EXECUTABLE --list-keys - # Install twine - pip install six pyopenssl ndg-httpsclient pyasn1 -U - pip install requests[security] twine wheel - # Execute the publish script for real this time - MB_PYTHON_TAG=$MB_PYTHON_TAG DO_GPG=True GPG_KEYID=$GPG_KEYID TWINE_PASSWORD=$TWINE_PASSWORD TWINE_USERNAME=$TWINE_USERNAME GPG_EXECUTABLE=$GPG_EXECUTABLE CURRENT_BRANCH=release DEPLOY_BRANCH=release DO_UPLOAD=True DO_TAG=False ./publish.sh - # Have the server git-tag the release and push the tags - VERSION=$($PYTHON_EXE -c "import setup; print(setup.VERSION)") - # do sed twice to handle the case of https clone with and without a read token - URL_HOST=$(git remote get-url origin | sed -e 's|https\?://.*@||g' | sed -e 's|https\?://||g' | sed -e 's|git@||g' | sed -e 's|:|/|g') - echo "URL_HOST = $URL_HOST" - # A git config user name and email is required. Set if needed. - if [[ "$(git config user.email)" == "" ]]; then - git config user.email "ci@circleci.com" - git config user.name "CircleCI" - fi - if [ $(git tag -l "$VERSION") ]; then - echo "Tag already exists" - else - git tag $VERSION -m "tarball tag $VERSION" - git push --tags "https://${PERSONAL_GITHUB_PUSH_TOKEN}@${URL_HOST}" - fi - ################################### ### INHERIT FROM BASE TEMPLATES ### @@ -568,31 +291,9 @@ jobs: - PYTHON_EXE: pypy3 - # -- gpgsign + deploy - - gpgsign/cp38-38-linux: - <<: *gpgsign_template - docker: - - image: circleci/python:3.8 - - - deploy/cp38-38-linux: - <<: *deploy_template - docker: - - image: circleci/python:3.8 - - .__doc__: &__doc__ - | - - # Test GPG sign works - load_secrets - circleci local execute \ - -e CIRCLE_CI_SECRET=$CIRCLE_CI_SECRET \ - --job gpgsign/cp38-38-linux - - IMAGE_NAME=circleci/python:3.9 docker pull $IMAGE_NAME @@ -669,8 +370,6 @@ jobs: JOB_NAME=test_minimal/pypy3 circleci local execute --job $JOB_NAME - circleci local execute --job gpgsign/cp38-38-linux - JOB_NAME=test_full/pypy3 circleci local execute --job $JOB_NAME diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..6a61862a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "friday" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..e5c365c2 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,365 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: Tests + +on: + push: + pull_request: + branches: [ master ] + +jobs: + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install flake8 + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 ./xdoctest --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=20 --max-line-length=127 --statistics + + build_and_test_sdist: + name: Test sdist Python 3.8 + runs-on: ubuntu-latest + #needs: [lint] + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Upgrade pip + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements/tests.txt + python -m pip install -r requirements/runtime.txt + - name: Build sdist + run: | + python setup.py sdist + - name: Install sdist + run: | + ls -al ./dist + pip install dist/xdoctest*.tar.gz -v + mv xdoctest ignore_src_xdoctest + - name: Test minimal sdist + run: | + pwd + ls -al + # Run the tests + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest --cov=xdoctest $XDOCTEST_DPATH ./testing + + - name: Test full sdist + run: | + pwd + ls -al + python -m pip install -r requirements/optional.txt + # Run the tests + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest $XDOCTEST_DPATH ./testing + + + - name: Upload sdist artifact + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./dist/*.tar.gz + + build_and_test_wheels: + # TODO: handle different python versions: https://github.com/actions/setup-python + name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }} + runs-on: ${{ matrix.os }} + #needs: [lint] + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macOS-latest] + python-version: + - '3.5' + - '3.6' + - '3.7' + - '3.8' + arch: [auto] + #cibw_build: [cp3*-*] + #cibw_skip: ["*-win32"] + # Add additional workers to reduce overall build time + #include: + # - os: windows-latest + # cibw_build: cp3*-win32 + # arch: auto + # cibw_skip: "" + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp35-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp36-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp37-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp38-* + # - os: ubuntu-latest + # arch: aarch64 + # cibw_build: cp39-* + + + steps: + - name: Checkout source + uses: actions/checkout@v2 + + # Configure compilers for Windows 64bit. + - name: Enable MSVC 64bit + if: matrix.os == 'windows-latest' && matrix.cibw_build != 'cp3*-win32' + uses: ilammy/msvc-dev-cmd@v1 + + # Configure compilers for Windows 32bit. + - name: Enable MSVC 32bit + if: matrix.os == 'windows-latest' && matrix.cibw_build == 'cp3*-win32' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + # Emulate aarch64 ppc64le s390x under linux + - name: Set up QEMU + if: runner.os == 'Linux' && matrix.arch != 'auto' + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml + #- name: Build wheels + # uses: pypa/cibuildwheel@v1.11.0 + # with: + # output-dir: wheelhouse + # # to supply options, put them in 'env', like: + # env: + # CIBW_SKIP: ${{ matrix.cibw_skip }} + # CIBW_BUILD: ${{ matrix.cibw_build }} + # CIBW_TEST_REQUIRES: -r requirements/tests.txt + # CIBW_TEST_COMMAND: python {project}/run_tests.py + # # configure cibuildwheel to build native archs ('auto'), or emulated ones + # CIBW_ARCHS_LINUX: ${{ matrix.arch }} + + # Setup Python environemtn for a pure wheel + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Build pure wheel + shell: bash + run: | + python -m pip install setuptools>=0.8 wheel + python -m pip wheel --wheel-dir wheelhouse . + + - name: Test pure wheel + shell: bash + env: + CI_PYTHON_VERSION: py${{ matrix.python-version }} + #OS_NAME: ${{ matrix.os }} + run: | + # Remove source directory (ensure it doesn't conflict) + mv xdoctest ignore_src_xdoctest + rm pytest.ini + # Install the wheel + python -m pip install wheelhouse/xdoctest*.whl + python -m pip install -r requirements/tests.txt + # Run in a sandboxed directory + WORKSPACE_DNAME="testdir_${CI_PYTHON_VERSION}_${GITHUB_RUN_ID}_${RUNNER_OS}" + mkdir -p $WORKSPACE_DNAME + cd $WORKSPACE_DNAME + # Run the tests + # Get path to installed package + XDOCTEST_DPATH=$(python -c "import xdoctest, os; print(os.path.dirname(xdoctest.__file__))") + echo "XDOCTEST_DPATH = $XDOCTEST_DPATH" + python -m pytest -p pytester -p no:doctest --xdoctest --cov-config ../.coveragerc --cov-report term --cov=xdoctest $XDOCTEST_DPATH ../testing + mv .coverage "../.coverage.$WORKSPACE_DNAME" + cd .. + # Move coverage file to a new name + + - name: Show built files + shell: bash + run: ls -la wheelhouse + + - name: Set up Python 3.8 to combine coverage Linux + if: runner.os == 'Linux' + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Combine coverage Linux + if: runner.os == 'Linux' + run: | + echo '############ PWD' + pwd + ls -al + python -m pip install coverage[toml] + echo '############ combine' + coverage combine . + echo '############ XML' + coverage xml -o ./tests/coverage.xml + echo '############ FIND' + find . -name .coverage.* + find . -name coverage.xml + + - name: Codecov Upload + uses: codecov/codecov-action@v1 + with: + file: ./tests/coverage.xml + + - name: Upload wheels artifact + uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/xdoctest*.whl + + deploy: + # Publish on the real PyPI + name: Uploading to PyPi + runs-on: ubuntu-latest + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + needs: [build_and_test_wheels, build_and_test_sdist] + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Download wheels and sdist + uses: actions/download-artifact@v2 + with: + name: wheels + path: dist + + - name: Show files to upload + shell: bash + run: ls -la dist + + # Note: + # See ../../dev/setup_secrets.sh for details on how secrets are deployed securely + - name: Sign and Publish + env: + TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ + TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} + TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} + EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} + run: | + ls -al + GPG_EXECUTABLE=gpg + $GPG_EXECUTABLE --version + openssl version + $GPG_EXECUTABLE --list-keys + echo "Decrypting Keys" + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import + echo "Finish Decrypt Keys" + $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" + $GPG_EXECUTABLE --list-keys + MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") + VERSION=$(python -c "import setup; print(setup.VERSION)") + pip install twine + pip install six pyopenssl ndg-httpsclient pyasn1 -U --user + pip install requests[security] twine --user + GPG_KEYID=$(cat dev/public_gpg_key) + echo "GPG_KEYID = '$GPG_KEYID'" + MB_PYTHON_TAG=$MB_PYTHON_TAG \ + DO_GPG=True GPG_KEYID=$GPG_KEYID \ + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ + TWINE_PASSWORD=$TWINE_PASSWORD \ + TWINE_USERNAME=$TWINE_USERNAME \ + GPG_EXECUTABLE=$GPG_EXECUTABLE \ + DO_UPLOAD=True \ + DO_TAG=False ./publish.sh + + test_deploy: + # Publish on the test PyPI + name: Uploading to Test PyPi + runs-on: ubuntu-latest + #if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) + if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags') + needs: [build_and_test_wheels, build_and_test_sdist] + steps: + - name: Checkout source + uses: actions/checkout@v2 + + - name: Download wheels and sdist + uses: actions/download-artifact@v2 + with: + name: wheels + path: dist + + - name: Show files to upload + shell: bash + run: ls -la dist + - name: Sign and Publish + env: + TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ + TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} + TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} + EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} + run: | + ls -al + GPG_EXECUTABLE=gpg + $GPG_EXECUTABLE --version + openssl version + $GPG_EXECUTABLE --list-keys || true + echo "Decrypting Keys" + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust + GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import + echo "Finish Decrypt Keys" + $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" + $GPG_EXECUTABLE --list-keys + MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") + VERSION=$(python -c "import setup; print(setup.VERSION)") + pip install twine + pip install six pyopenssl ndg-httpsclient pyasn1 -U --user + pip install requests[security] twine --user + GPG_KEYID=$(cat dev/public_gpg_key) + echo "GPG_KEYID = '$GPG_KEYID'" + MB_PYTHON_TAG=$MB_PYTHON_TAG \ + DO_GPG=True GPG_KEYID=$GPG_KEYID \ + TWINE_REPOSITORY_URL=${TEST_TWINE_REPOSITORY_URL} \ + TWINE_USERNAME=${TEST_TWINE_USERNAME} \ + TWINE_PASSWORD=${TEST_TWINE_PASSWORD} \ + GPG_EXECUTABLE=$GPG_EXECUTABLE \ + DO_UPLOAD=True \ + DO_TAG=False ./publish.sh + + +### +# Unfortunately we cant (yet) use the yaml docstring trick here +# https://github.community/t/allow-unused-keys-in-workflow-yaml-files/172120 +#__doc__: | +# # How to run locally +# # https://packaging.python.org/guides/using-testpypi/ +# cd $HOME/code +# git clone https://github.com/nektos/act.git $HOME/code/act +# cd $HOME/code/act +# chmod +x install.sh +# ./install.sh -b $HOME/.local/opt/act +# cd $HOME/code/xdoctest + +# load_secrets +# unset GITHUB_TOKEN +# $HOME/.local/opt/act/act \ +# --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \ +# --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \ +# --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \ +# --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \ +# --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD diff --git a/CHANGELOG.md b/CHANGELOG.md index bf12a9e3..8af49bc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,15 +4,24 @@ We are currently working on porting this changelog to the specifications in [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Version 0.15.8 - Unreleased +## Version 0.15.9 - Unreleased + +### Changed + +* Added GitHub actions to the CI +* Disabled workaround 16806 in Python 3.8+ +* New CI GPG Keys: Erotemic-CI: 70858F4D01314BF21427676F3D568E6559A34380 for + reference the old signing key was 98007794ED130347559354B1109AC852D297D757. ### Fixed -* Hotfix - removed debug print statements +* Fixed minor test failures +* Fixed #106 - an issue to do with compiling multiline statements in single mode. +* Fixed #108 - an issue to do with compiling semicolon token in eval mode. -## Version 0.15.7 - Released 2021-09-02 +## Version 0.15.8 - Released 2021-09-02 ### Changed * Removed the distracting and very long internal traceback that occurred in @@ -22,13 +31,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm nothing unless `-s` is also given so pytest does not supress output) +## Version 0.15.7 - Yanked + ### Fixed * Bug in REQUIRES state did not respect `python_implementation` arguments * Ported sphinx fixes from ubelt ## Version 0.15.6 - Released 2021-08-08 - ### Changed * Directive syntax errors are now handled as doctest runtime errors and return better debugging information. @@ -168,7 +178,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Changed * `PythonPathContext` now works in more corner cases, although some rarer corner cases will now break. This trade-off should be a net positive. -* Releases are handled by TravisCI and will be signed with the GPG key 98007794ED130347559354B1109AC852D297D757 (note we will rotate this key in 1 year). +* Releases are handled by TravisCI and will be signed with the GPG key 98007794ED130347559354B1109AC852D297D757 (note we will rotate this key in 1 year). <- (2021-09-06) lol that did not happen, dsomeday I'll get around to setting up rotating GPG keys. ## [Version 0.10.0] - Released 2019-08-15 @@ -180,8 +190,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm * Add `--version` option to CLI interface ### Changed -* Improved backwards compatibility. Explicit continuations now work more similarly to the original doctest. -* You no longer need a comment to denote that a `...` is a continuation and not a ellipsis. +* You no longer need a comment to denote that a `...` is a continuation and not + a ellipsis. (i.e. you don't need to write `... #`) * Want statements will check against return values in nested continuations * Cleaned up internal code, private APIs may break. * Failed doctests will now print their original line prefixes (either `>>> ` or `... ` when possible) diff --git a/README.rst b/README.rst index b6616acc..1c3e5ffc 100644 --- a/README.rst +++ b/README.rst @@ -268,8 +268,9 @@ The main enhancements ``xdoctest`` offers over ``doctest`` are: 2. Additionally, the multi-line strings don't require any prefix (but its ok if they do have either prefix). 3. Tests are executed in blocks, rather than line-by-line, thus - comment-based directives (e.g. ``# doctest: +SKIP``) are now applied - to an entire block, rather than just a single line. + comment-based directives (e.g. ``# doctest: +SKIP``) can now applied + to an entire block (by placing it one the line above), in addition to having + it just apply to a single line (by placing it in-line at the end). 4. Tests without a "want" statement will ignore any stdout / final evaluated value. This makes it easy to use simple assert statements to perform checks in code that might write to stdout. @@ -278,11 +279,10 @@ The main enhancements ``xdoctest`` offers over ``doctest`` are: 6. Ouptut from multiple sequential print statements can now be checked by a single "got" statement. (new in 0.4.0). -See code in ``_compare/compare.py`` and ``_compare/base_diff.py`` for a demo -that illustrates several of these enhancements. This demo mostly shows cases -where ``xdoctest`` works but ``doctest`` fails, but it does show **the only -corner case I can find** where ``doctest`` works but ``xdoctest`` does not. -Feel free to submit more in an issue if you can find any other backwards +See code in ``dev/_compare/demo_enhancements.py`` for a demo that illustrates +several of these enhancements. This demo shows cases where ``xdoctest`` works +but ``doctest`` fails. As of version 0.9.1, there are no known syntax backwards +incompatability. Please submit an issue if you can find any backwards incompatible cases. @@ -393,7 +393,7 @@ Thus, a test that fails in ``doctest`` based on a "got"/"want" check, may pass in ``xdoctest``. For this reason it is recommended that you rely on coded ``assert``-statements for system-critical code. This also makes it much easier to transform your ``xdoctest`` into a ``unittest`` when you realize your -doctests start getting too long. +doctests are getting too long. diff --git a/dev/_compare/base_common.py b/dev/_compare/base_common.py deleted file mode 100644 index 75aa4f52..00000000 --- a/dev/_compare/base_common.py +++ /dev/null @@ -1,6 +0,0 @@ -def do_asserts_work(): - """ - >>> # xdoctest: +REQUIRES(--demo-failure) - >>> assert False, 'this test should fail' - """ - pass diff --git a/dev/_compare/compare.py b/dev/_compare/compare.py deleted file mode 100644 index c41b66fd..00000000 --- a/dev/_compare/compare.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Compare xdoctest with doctest - -This file autogenreates two files: compare_doctest.py and compare_xdoctest.py -They will have the same body up until the main block as defined in -base_diff.py. One will run doctest and the other will run xdoctest. See the -difference. -""" -import ubelt as ub - - -def generate(): - content = ub.readfrom('base_diff.py') + '\n\n' - xdoc_version = content + ub.codeblock( - ''' - if __name__ == '__main__': - import xdoctest - xdoctest.doctest_module(__file__) - ''') + '\n' - - doc_version = content + ub.codeblock( - ''' - if __name__ == '__main__': - import doctest - doctest.testmod() - ''') + '\n' - - ub.writeto('_doc_version.py', doc_version) - ub.writeto('_xdoc_version.py', xdoc_version) - - -def main(): - generate() - # Run the files - - print('\n\n' + ub.codeblock( - ''' - ___ ____ ____ ___ ____ ____ ___ - | \ | | | | |___ [__ | - |__/ |__| |___ | |___ ___] | - - Cant do most of this in doctest, although apparently you can - use asserts, whereas I previously thought they didnt work - ''') + '\n\n') - - ub.cmd('python _doc_version.py', verbose=2) - - print('\n\n' + ub.codeblock( - ''' - _ _ ___ ____ ____ ___ ____ ____ ___ - \/ | \ | | | | |___ [__ | - _/\_ |__/ |__| |___ | |___ ___] | - - Just run the assert failure to illustrate how failures look - ''') + '\n\n') - ub.cmd('python _xdoc_version.py do_asserts_work --demo-failure', verbose=2) - - print('\n\n' + ub.codeblock( - ''' - _ _ ___ ____ ____ ___ ____ ____ ___ - \/ | \ | | | | |___ [__ | - _/\_ |__/ |__| |___ | |___ ___] | - - Run all other tests, to show how the ast based xdoctest can deal with - syntax that regex based doctest cannot handle. - ''') + '\n\n') - ub.cmd('python _xdoc_version.py all', verbose=2) - - -if __name__ == '__main__': - """ - CommandLine: - python ~/code/xdoctest/_compare/compare.py all - python xdoc_version.py all - python doc_version.py - """ - main() diff --git a/dev/_compare/base_diff.py b/dev/_compare/demo_enhancements.py similarity index 66% rename from dev/_compare/base_diff.py rename to dev/_compare/demo_enhancements.py index c07d03b3..09540842 100644 --- a/dev/_compare/base_diff.py +++ b/dev/_compare/demo_enhancements.py @@ -1,8 +1,12 @@ """ -TODO: - Anything that works in both should be moved into a different files to show - commonalities, and anything that only works in one should be in another - file to show differences. +This file contains doctests that work in xdoctest but fail in doctest + +Use the following command lines to run the doctest and xdoctest version to see +the difference: + +CommandLine: + python -m xdoctest demo_enhancements.py + python -m doctest demo_enhancements.py """ @@ -29,8 +33,6 @@ def embeded_triple_quotes(): """ pass -# TODO: fix the higlighting of the "got" string when dumping test results - def sequential_print_statements(): """ @@ -54,7 +56,7 @@ def repl_print_statements(): def multiple_eval_for_loops_v1(): """ - This is one corner case, where doctest can do something xdoctest cannot. + Previously this failed in xdoctest, but now it works as of 0.9.1 >>> for i in range(2): ... '%s' % i @@ -74,3 +76,20 @@ def multiple_eval_for_loops_v2(): 0 1 """ + + +def compact_style_code(): + """ + This compact style is a bit ugly, but it should still be valid python + + Exception: + >>> try: raise Exception # doctest: +ELLIPSIS + ... except Exception: raise + Traceback (most recent call last): + ... + Exception + ... + + """ + try: raise Exception # NOQA + except Exception: pass # NOQA diff --git a/dev/_compare/demo_failures.py b/dev/_compare/demo_failures.py new file mode 100644 index 00000000..5af86dad --- /dev/null +++ b/dev/_compare/demo_failures.py @@ -0,0 +1,26 @@ +""" +This contains that fail in both. This demos what correct +failures look like in each case. + +CommandLine: + python -m xdoctest demo_failures.py + python -m doctest demo_failures.py +""" + + +def do_asserts_work(): + """ + >>> assert False, 'this test should fail' + """ + pass + + +def multiple_eval_for_loops_v1_fail(): + """ + + >>> for i in range(2): + ... '%s' % i + ... + 0 + 1 + """ diff --git a/dev/_compare/demo_issue_106.py b/dev/_compare/demo_issue_106.py new file mode 100644 index 00000000..397494bb --- /dev/null +++ b/dev/_compare/demo_issue_106.py @@ -0,0 +1,114 @@ +r""" + https://github.com/Erotemic/xdoctest/issues/106 + cd ~/code/xdoctest/dev/_compare/ + python -m xdoctest demo_issue_106.py + python -m doctest demo_issue_106.py + +Note: + the reason this fails is because this fails: + + compile('try: raise Exception\nexcept Exception: print', mode='single', filename="") + + + In exec mode we are ok + + compile('try: raise Exception\nexcept Exception: print', mode='exec', filename="") + + This has to do with the assign ps1 line function that determines if + we should be in exec or single mode + + Other tests + + + compile('if 1:\n a', mode='single', filename="") + compile('if 1:\n print', mode='single', filename="") + compile('if 1:\n x = 1\n y = 2\nelse:\n pass', mode='single', filename="") + + compile('try:\n raise Exception\nexcept Exception:\n pass', mode='single', filename="") + compile('try: raise Exception\nexcept Exception: pass', mode='single', filename="") + + except Exception: print', mode='single', filename="") + +""" +import sys + + +def logTracebackThisDoesnt(logFunction): + r""" Logs the exception traceback to the specified log function. + + >>> # xdoctest: +IGNORE_WANT + >>> try: raise Exception() # doctest: +ELLIPSIS + ... except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) + Traceback (most recent call last): + ... + Exception + ... + """ + sys.exc_info() + # import xdev + # xdev.embed() + logFunction + pass + + +def slurpFile(path, mode, maxbytes, **kwds): + """ + >>> import os; slurpFile(os.path.abspath(__file__), mode = 'rb')[:9] + b'# coding=' + >>> import os; slurpFile(os.path.abspath(__file__), encoding='utf-8')[:9] + '# coding=' + """ + pass + + +# def logTracebackThisWorks(logFunction): +# r""" Logs the exception traceback to the specified log function. + +# >>> try: raise Exception() # doctest: +ELLIPSIS +# >>> except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) +# Traceback (most recent call last): +# ... +# Exception +# ... +# """ +# sys.exc_info() +# # import xdev +# # xdev.embed() +# logFunction +# pass + +# # ... except Exception: logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) + +# # def compact_style_code(): +# # """ +# # This compact style is a bit ugly, but it should still be valid python + +# # Exception: +# # >>> try: raise Exception # doctest: +ELLIPSIS +# # ... except Exception: raise +# # Traceback (most recent call last): +# # ... +# # Exception +# # ... + +# # """ +# # try: raise Exception # NOQA +# # except Exception: pass # NOQA + + +# def logTraceback2(logFunction): +# r""" Logs the exception traceback to the specified log function. + +# >>> try: +# ... raise Exception() +# ... except Exception: +# ... logTraceback(lambda *a, **b: sys.stdout.write(a[0] + "\n", *a[1:], **b)) +# Traceback (most recent call last): +# ... +# Exception +# ... +# """ +# import sys +# logFunction(*sys.exec_info) +# logFunction() +# pass diff --git a/dev/ci_public_gpg_key.pgp.enc b/dev/ci_public_gpg_key.pgp.enc index f99f82ca..0ac8eadd 100644 --- a/dev/ci_public_gpg_key.pgp.enc +++ b/dev/ci_public_gpg_key.pgp.enc @@ -1,16 +1,35 @@ -U2FsdGVkX1/NUJtlVaXysoiU5E0+zVqAA7zqbXzkF0IM7XgzVP2h7yWJ3GE9zD23 -h9RQpwi8Vl5bSOvZV4n/lOEnhSV2pelHZOGet80ewbaKeU+Trp8Gv3pU/l9FZrAx -EU0jk5oEojHnFnC0H/uAC3qeQ8WWmpGXklpPdbMu2NiOhDFrAhAMIDkNxL5L7Rtt -DnqO5em/6iCatkTCeA9pQU7/vjV3OHNzf9XjjBcq+bOPge1hup1gJ3/TpC1ughr8 -2cYoXNaDkXnktA3NU33S94VVAXYpQVH1vMS95NBCjeru//ktf+QUv/4TuXxTU899 -N/5sQH/OP8qs+/h3+/wG6ea+A6j8G4+sgAp4HSkxazuu0zUvufjciuYg4C8lPZtl -/2GiEF78whVAgReeGvlYA7hOfGDNXfvhLKRN4QK5vE2QmJMQW7nvh0arvaBm9QCa -CaTTFf34AQkDmBounuBn0tb3bJQlaZJ1yZQhstsOJfvLJUKPHKZmIvO6mfNcaeLL -qFDCN3VU+z0IpuFucS5kr0A3IyuVr38B/E4rw9ilwfwJfbPrAI6imxa2HVXUVzWi -eOEFIlF5TFgMnY5+yKUn8YbpcVZRUbI0wuFcYFZvKDHlRMP8FibBdphyfsTujM84 -inTBMy5NJoaDDnVUShQUuiQovxGJSdttFLAoQPaUpqs9E7l5a+4pdcB/jametlgZ -ky21IogZsuQ8Hzrp2tZDk0P64qu3vCse2/hgNsBrewIkT5OpT47ZKKSzcjZEl+Od -/1Ok2glPc9Qg6lwuQT0knHVPAyl1K36ejdLHHibUiKnGXxXr1wq52OA/wmFCVzxs -J46NcSVYAOsmqp7lKWYDyHgEDnBmBAM3oyAL5rxniFhCxUBlGW0IGUBpdMR0IDr9 -btaWF/1NCJIFTExmaH+9WuFPbkLvmSKUdwIvlfzG2I5RskIxg1zRtNIO9BoQOd/r -2e9zO+zVgDBVXaBCDBtfA1+s0dtPkOnlE4O1WBP8ShqDQvs5IcmXYXJPPMiSikg7 +U2FsdGVkX1+X4uezbbjDmxrEhCujtGYg5wZPLKLZd8hN1AfAI6kahNjAMQpBfehc +QzzWfdeE1gaeGUaigghQXjYOz4cNgC43ti0HiI6o5lCu9QgTDq5NOy8BZVyZvb6G +i57Z4OpwA2wU4i+QgUOA5aMifGEc0MeRzBMn0N6mAbxhgA3YJEyknatg3iTjlt5t +knmGNCu1fa2aDO/DvYtwRuIISDu7lCUFrw1Fi9riXGteoN4FZzju+bN2S4bQfjSe +WI156ZSTg9A5Os93u3Z6xofzchgUTnqAfNkxSLxbl8Jk5AEvZnP0YtLphcl9Hdgm +nKYs7v0sSSBdVeItFB2lOYc2iq5CiUxRPvXb4I3G47grEgBwqeKcvixb6dEs7X1N +/utcnTBFqjB+pa7LTH+scuNwqc7z5pJ1F0bY6fwmOwd8P8X2WTdDFDhiJx2lyrhu +zvWE1Ppu1iuQW8T9Bg8bLkyvtKjHCJTKSCVUHoEH+TpGHaUcI9M0XC4hJKIGeRVM +uTIUNhN3sOS0uZ6jjN17cyhj+9SAzkWgRiqoiHd/Hzs3XqPqTesgJCMUsQwSrgxz +pHCSz12rW4U0SdwHF7X8jFLMVYSD7VA73nPxEgwgGfBWuxOAb2oKj/iXkyY53nDB +SgWZ24d5i/SkGsz8cJRcu7zhhHki4rhiijG5dU/B7XLQGXpbiKmB/eWt2kdmdy6E +bcJNIqQWq3T02wV1rottXssf3OAL/kC99ZlPm3HcR//kiMQQ7sWR0lPJECA1Inyt +2nuTTxMd4LAq1lL/zVwAf7a0N1miHq3yCe1nW3elGUwwwdS4Q4YwrRIUrRNd+PmE +QTD7ki/pIJ0mi5N8wAwXiAsTUUtWC0eCOZLwft9lulrDZfWM1zJ/zrqLhIhUuX2W +xAAInK6UuJFzPb5+SKAzuNOBSOyM8GChro0Da9UKZgirp7GY8+0/mmKCCkAofZEZ +A8yvx+wZCzeNsW6+ZwH/CfGDcBW1wwtiEdTGeUpHsAoz6+GK9PvgbJFzHcmxJ5ng +GsDs1WpILXBo972sxteyEyyOFOHVxRvXFJsrA1o3O49DouHf80K8nKJcPDIFuC6L +kdFjEpvvRAUXTtJdjxJqpPh1aR4iiHQwkDyM+3WY36o/O+iRyXoJdltCZF96mBrj +K0yhVe+g4EulOhjAr+mKWxu9+n1TX2k6GUZrnh0Mux31ksNvoxqwhf9K+E1uCFDW +JKpOdqSlJBj+w2hTKlLp1VH6rw9aIh1YCmHBQSiPsQvqeXkvmbZtz7ZotlianO7K +f3lgBxNnvpGfokWZ8ZubUICo3rG7lwD1FZV2+AEQWya03noF1M7FVRCzb52DloSl +BeCnxxGe3PhZcSMqqrMltKETk+QcBLUMEoOqTKLMvihJ1VDk+rKAmEB55EV1qhli +DJrxaH0VSK3wzLKOkmmDeRRSyvMPtIpO8vuCpQQqjEVo+Du0BPfi/ItdXA/GijXu +RMv60zYzZUCXz2f/FVOsNVq+iugd4Z/6QA35P3LMizjDjiwSza/otNQSQCGUVdpc +7/uVGAY3v/nDN1xlP5HtVZUleRTKcvw/uu/077szAL6tXSLBRwo8zH4wZ3uavyAC +b9jg92zoBIMsHxITc03Vih2d+L4qEbWuyUaNxFW6vmbk22P7zl+8uGSWYu9PqdEO +/985GXfzS5eAMW/N3hgkVHYEjH7WdudhSI8FImBvrurlg9/qm/WaxqPGTg+MFU1v +bAp9uRV2XZuSlmeT4BKadYt7N5lDnmgueF5q3hErTPghABzdL9VFA+8KebJU6IoH +bJxZbkRP+QQCOOYGlyWFICxTxorESOZt4izqsXu+Bwryjt0FzFYpcvo2XIzEn4Yz +alw1I8Xcy0aVLdffNeAmtskeshitIBMAb44mH2UEmTZ/+ITIK/zFnpkXipg+6p8P +u0P0MExTALCDee0KcOC3czgIqtv/uQ8M4ymi8nZNEdOdDyjl4ugwA1QAY6OVQLnj +RnUn46jgotEFdF0aNMcXdylR1zuboDNpYBwuWNpzVn0k3zTaslB2QrJk4saoTJfh +eFuY1Y8gTDugDwtSUo+n+OyiZpfykUv5ASIfwjz4XufR2FCT3GNOxs9CbACSEZkJ +dptkzX6JeUhxXxP/zQYn3MisdWtjmpPKPBz1irwnKOeYUQYYUrD1YhoJo+JAUWVh +2t9jh8KHphkD0FNM7hjaAg== diff --git a/dev/ci_secret_gpg_key.pgp.enc b/dev/ci_secret_gpg_key.pgp.enc deleted file mode 100644 index 9cfa93a8..00000000 --- a/dev/ci_secret_gpg_key.pgp.enc +++ /dev/null @@ -1,18 +0,0 @@ -U2FsdGVkX1/cIIJK5fbB8daDmOnQwrhsHJjiSk5hv92Z/WKR3zTe6KG9Sktd9RTK -gVSoMBbPq/tJ5e6i0niTjouAvjdJTPP2Uc6nupduV5L/Qw5MdeQEz66eoj6e1Y5T -TIjs61LEt7AbIMnO3ohBpKEBzbIlsAYYJ7q8N4FaWLw9rLImpcibWCXW6YYiSbPI -mFbP1GMhmNqKKSV5ccMddGQjYk22DI2uSVWaV2xz82wbil02wcw7AJfD+5hoK5i+ -7Q9vBAbtt1RCw+P+O1JQ0cAmbBoMCZkL0CH5CIlnncCHf/l8vHUYppwp4fUcueW4 -AXlrVNuSdJW26jBn9YcSgWjCrtO7Uv1qZwwjz4YAaGJD0nmSIiQweXvoa3CgQrx7 -r/bln8W92vNmBy2kPmUbA1Ory3MTkw5k9+1RFnxPF+Z9OOzZW82Abt+yKW5HsA5D -fzNgfOsMY8r8ydH+oqf+sWJMO4xEZW0LZYTyZ7HiuDjWhHQABFnTFteM1eBM02yg -SzfpfJMQukf0qV8eVvgI2UjiTuengQwaAr4GPF7gs3dIZhNdZS2+zKCBIQR+fe4a -86ORIqwJxNxhwchzH47cDxDEI3By8TOCyQHZGAiOTeehdmbs9Ay+qJuWjOnUwh36 -1ACraJuMa+6btq11G2rMKNPqhQw6mGONUKPNKvYDL6lGzwUXD+WxYihFy01WnB7J -ot6wUYZDno3ut5IczcIQMR4T/D06JdGJSCsjbashTn3772Xv1uwGQ5WEoeGalcJt -OvidnnVODoPgRL03hKoeM41bwePoZGDVOg362vcw1ZlXIocnTq0tpxFaDcXU1+OP -OiqXXz62xOEMhKqDVsUU1pjG3m4pudNWy0zJEbMtt/Z1gH3mJZas7DaFsHEA53S1 -yvIN0b3uESYUlOAiRv7Fz3zx09HtPtO42anamyuCA9zdaru2msISj8ORTq7g7dH8 -pULT7K9WmXcxi3XpCan9FcQTQJCcFvOrL7jAW+BisP5RFgeH2tM5FviSI+95u1B2 -Uh00XC6qT+NaHgzJS9hvajouojV3GjdTocUSHby+5P8at4CWNiwDHlBlrusI5vb7 -XAYVynRrQVKGtnnjUHRhWJr9P89MYHx+NIaI2tnOxWRrO2AT/50dkp7cru6gh0rh diff --git a/dev/ci_secret_gpg_subkeys.pgp.enc b/dev/ci_secret_gpg_subkeys.pgp.enc new file mode 100644 index 00000000..3d37c446 --- /dev/null +++ b/dev/ci_secret_gpg_subkeys.pgp.enc @@ -0,0 +1,27 @@ +U2FsdGVkX18p5fLvjvY0CGnafpqx3WPqzfkIY8I/AIM4aCEjzz2DldkFUoYYwGLc +FB4E10U1cxIByCK9Nk2aSjhognaIP4IOWCwD7ti1L0zy0CuZdlzgWVdH26xs6EZ8 +TKJD1STS/oexppTzJFbHUC5MqR/wMSNYFK/OQbhEjidxsrRH2P4NSQiYq34s3vU4 +JJsjnQUREdDYgctcCempI0FkPC/aAYkkS14j+iBnvQ6gM/OnovhTP/4iPLhraTCq +qYFcP0cdQ9V1puEOnxSCA7qNXMynwE5HvSft/Q+/+1RhMgMlKaXrd3Hhcs+nxrxO +X8zlSI63b2Q7K4XIQ2RpTnirUVSrN//WE5TOSY9GC+XDm6VSJ6ncAf7qWB0Cvycs +o51DbzT8iGflRXYH0PtcLkyD/3dBU4WaZFT9HPmpWHT51oh7i6gjJxDaC8YO8GGd +2VMnFvXGH1goqIThLIkmqqjyWBCinrWsxpBOnh9sgpOUIV5GdrucJb9kOSC+jCt6 +E4nML+R3f8OZt747gAmA+LAFHAPCYt+pHzXN06XPan5lUe+tkMjeLWhJN6DE2hy/ +AFCjoXXNtvq0WzPFaqw19ryLMsvH6ocZz3tOIJgBOVWFLbkG0uwFZSWKF+zmedsT +OtYM9QS87fdoIt5FHHs0Ic4/v3SzBXPz1JJDbGanqv5VC8NTFKle7YqlQAmi2/Mx +OsYxUGa2Zw5hU7S4TZgod0j/Ip/fAUeG43o4Kh+br0ivCYf0RO/LgUbC0l9oEGco +MzE5vxyh++21RnxhlZ+vdcU+KY5EOCwor43TodM7RShYGF7+YwpeA8CpfC85VQWw +18KIWxfCed0SM5kDKpZ0ISNwqofL1H0tBPYP9X9it5v0U0zhB9m5Gr7eS2Hqu1dk +Z3q0Kz8CbflLx8DNQ6MqR5iD63G5EwGqPtb7RBgBRZ0vnG1yXl52yTocI4aO6Iyt +kq/DOvyp4JVc5xAvvajstt87eYyST0konklSE/uSTxYDRcSuOSeP3VdP0K1E5H/L +JshVis++7cULrEVFa6/00ojsmAr2eU9dcWMy/9U1J/PNtUAi6JhVCZy1cIm2F4z2 +f4pOuey/C4Fs2izRObQ+v4lgfG0nJiIuJeovYB3qZ5/+P4HKOkArMIUUdO3QG67f +T7Y2x4BmcalE7fVtNhb4LxCfdqu/ku7jpY1VdkBnTgjno4E+2OSYbySbkGSlF+XR +lFPJToa3lSHf734rzIRyKcfRfccycGvdy5vIiSmWa8XryFZPw/BSHncv2p9YyCvC +JNsykxc4HHxx/AoYA4hlqDETz7PwaBoEf00yr3psL3486UWZvrIAzLsv9Fw/pwM7 +o8vKt2U7QnLnp5HuuoBEpqi3bUazMyWCoV2TfDZ+3BQYlmtWgpHE4TzN0VD3R6qB +mwobN58NX79ZS/U8cFcPxWzRCdWqv7Zd8KWtNIfN8SD28KamuY3fUvSbFId5v7ZS +uI3KZZl7yos0LHuS1jWBRqUZPNEfoFr2xg2PHZy28DewECxURCZADd8CcWPQoC9u +L+1yKxS8oD1Y/hmCg1fNjyX2kS3ZDNATtZX3mnMxkMNF/wmhLBNdmbkT99hjdj8o +3y7dpftOIC/njCEYWFlIy/eF3F7J+UUOJ4b1zSs1w5lTnTje4uTZ9Z5dAnLO1Myu +JE7E57qVJdlVBwtFYSJtjQVoYqNZjEX4+aAEFv2klAW+8quHJI9AvC0ph3YUw8si diff --git a/dev/gpg_owner_trust b/dev/gpg_owner_trust new file mode 100644 index 00000000..d51405be --- /dev/null +++ b/dev/gpg_owner_trust @@ -0,0 +1,9 @@ +# List of assigned trustvalues, created Thu 23 Sep 2021 09:31:07 PM EDT +# (Use "gpg --import-ownertrust" to restore them) +98007794ED130347559354B1109AC852D297D757:6: +2B9F0ED4BB49F26164A8B45DD6DBFC409CB9B78B:6: +F7C1C97A4EF2CF9643D94C666CC05F155B12AB36:6: +A1198702FC3E0A09A9AE5B75D5A1D4F266DE8DDF:6: +A490D0F4D311A4153E2BB7CADBB802B258ACD84F:6: +4AC8B478335ED6ED667715F3622BE571405441B4:6: +4AED9D3F16DB488FD7F33926C6AA1A4F42B93ED9:6: diff --git a/dev/gpg_owner_trust.enc b/dev/gpg_owner_trust.enc index ffe6d69c..2c2ea1a4 100644 --- a/dev/gpg_owner_trust.enc +++ b/dev/gpg_owner_trust.enc @@ -1,7 +1,10 @@ -U2FsdGVkX19+WoDVlFCHUeo0irIBBe8Sp9N1YAi5DhAkKYd6M+C1YnwcSjQJaLvj -eM+HWcyBZfj1mN537jMmnrgS7cxGh4QvMDBMVifw1UYMGCcn/Rf+b2+0BymcGuLo -c3JEmdFXjIw2a0eHh8seBFq+HwLoU3bymz7Lr0XQK7Y6hPPSm7Cwo1HjOwuDXFa1 -RIutCuQC79og+sryYfQPBTEiCeJW0j0maUpS9H5gfnqXafp1+Fg7if/AskuATMXG -dUtkIYFJXwgY5qvD1pIQnmC/rKAZj5o8x8u2XewgVX1PHk6nrBI1dBUMnkRs2h8N -p6dxj+yC68YSUpdlPIvLdWxihOLjIlUbkdlpIFSO7hNbuKfb22On/TFHMKYeqN7Y -IGdcNCFiFaUkZWiTI6Nt6/6HgZNFrhqVjtGZDV7pcwM= +U2FsdGVkX19vZXMNVobiorDi2gogjVIjZ1ExKRsQlJ2k1xGjqffueVAJ/cy7DHLr +wsKKW8BKRErz68v/+P1+GgAhG/GPQdQCbuW2fAX/BZ+bteBDN1hPGd+YCX/C69NU +RtrqtcFrZHpuhTBE1fulBuzeLNmRaMEcDTjxFpkZ6hy1fTOZaSzrjAH+2/xcBkm/ +xElyfTSsJmbhOjpPTFtj6JVoRI++t2okNNzVfFs1OtGiPglFxjChan+LaacCmggS +WnJtk0mOq4LKZeq5GQBVwgcIy15p4GxU1AofBcUIIkWOcwq++s1rFKJOgCASKUFQ +CGWFQ7/0CffqxNNmrKzMM3BJXKF2XP9bT49hlGqpQJpCD9o5IRB77U5Vo83xJOft +QUnTghHKGkZ7v0WRAFGme3OqtY9+rCKcGq9aNzR+qPkapwstj4pjAaoETSr6V3tX +ORU4xqyo5EGURS9zK1d2lW4t5Y9LKxX0mjWbCVtPwdAHEOfDuA0fpVnoxQZ5vBp8 +qEA3u5V/bjRN8KP/07SNADHZibs+s1DiStCvbdyexNiOBLuYEime0qpgrz180NNQ +m5m9EsxHaBsZSnJcKO3j1Q== diff --git a/dev/public_gpg_key b/dev/public_gpg_key index debca485..91ee9e69 100644 --- a/dev/public_gpg_key +++ b/dev/public_gpg_key @@ -1 +1 @@ -D297D757 +70858F4D01314BF21427676F3D568E6559A34380 diff --git a/dev/secrets_configuration.sh b/dev/secrets_configuration.sh new file mode 100644 index 00000000..09933e9d --- /dev/null +++ b/dev/secrets_configuration.sh @@ -0,0 +1,2 @@ +export VARNAME_CI_SECRET="EROTEMIC_CI_SECRET" +export GPG_IDENTIFIER="=Erotemic-CI " diff --git a/dev/setup_secrets.sh b/dev/setup_secrets.sh new file mode 100644 index 00000000..ddda23a7 --- /dev/null +++ b/dev/setup_secrets.sh @@ -0,0 +1,211 @@ +__doc__=' +============================ +SETUP CI SECRET INSTRUCTIONS +============================ + +TODO: These instructions are currently pieced together from old disparate +instances, and are not yet fully organized. + +The original template file should be: +~/misc/templates/PYPKG/dev/setup_secrets.sh + +Development script for updating secrets when they rotate + + +The intent of this script is to help setup secrets for whichever of the +following CI platforms is used: + +../.github/workflows/tests.yml +../.gitlab-ci.yml +../.circleci/config.yml + + +========================= +GITHUB ACTION INSTRUCTIONS +========================= + +* `PERSONAL_GITHUB_PUSH_TOKEN` - + This is only needed if you want to automatically git-tag release branches. + + To make a API token go to: + https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/creating-a-personal-access-token + + +========================= +GITLAB ACTION INSTRUCTIONS +========================= + + ```bash + cat .setup_secrets.sh | \ + sed "s|utils||g" | \ + sed "s|PYPKG||g" | \ + sed "s|travis-ci-Erotemic||g" | \ + sed "s|CI_SECRET||g" | \ + sed "s|GITLAB_ORG_PUSH_TOKEN||g" | \ + sed "s|gitlab.org.com|gitlab.your-instance.com|g" | \ + tee /tmp/repl && colordiff .setup_secrets.sh /tmp/repl + ``` + + * Make sure you add Runners to your project + https://gitlab.org.com/utils/PYPKG/-/settings/ci_cd + in Runners-> Shared Runners + and Runners-> Available specific runners + + * Ensure that you are auto-cancel redundant pipelines. + Navigate to https://gitlab.kitware.com/utils/PYPKGS/-/settings/ci_cd and ensure "Auto-cancel redundant pipelines" is checked. + + More details are here https://docs.gitlab.com/ee/ci/pipelines/settings.html#auto-cancel-redundant-pipelines + + * TWINE_USERNAME - this is your pypi username + twine info is only needed if you want to automatically publish to pypi + + * TWINE_PASSWORD - this is your pypi password + + * CI_SECRET - We will use this as a secret key to encrypt/decrypt gpg secrets + This is only needed if you want to automatically sign published + wheels with a gpg key. + + * GITLAB_ORG_PUSH_TOKEN - + This is only needed if you want to automatically git-tag release branches. + + Create a new personal access token in User->Settings->Tokens, + You can name the token GITLAB_ORG_PUSH_TOKEN_VALUE + Give it api and write repository permissions + + SeeAlso: https://gitlab.org.com/profile/personal_access_tokens + + Take this variable and record its value somewhere safe. I put it in my secrets file as such: + + export GITLAB_ORG_PUSH_TOKEN_VALUE= + + I also create another variable with the prefix "git-push-token", which is necessary + + export GITLAB_ORG_PUSH_TOKEN=git-push-token:$GITLAB_ORG_PUSH_TOKEN_VALUE + + Then add this as a secret variable here: https://gitlab.org.com/groups/utils/-/settings/ci_cd + Note the value of GITLAB_ORG_PUSH_TOKEN will look something like: "{token-name}:{token-password}" + For instance it may look like this: "git-push-token:62zutpzqga6tvrhklkdjqm" + + References: + https://stackoverflow.com/questions/51465858/how-do-you-push-to-a-gitlab-repo-using-a-gitlab-ci-job + + # ADD RELEVANT VARIABLES TO GITLAB SECRET VARIABLES + # https://gitlab.kitware.com/computer-vision/kwcoco/-/settings/ci_cd + # Note that it is important to make sure that these variables are + # only decrpyted on protected branches by selecting the protected + # and masked option. Also make sure you have master and release + # branches protected. + # https://gitlab.kitware.com/computer-vision/kwcoco/-/settings/repository#js-protected-branches-settings + + +============================ +Relevant CI Secret Locations +============================ + +https://github.com/pyutils/line_profiler/settings/secrets/actions + +https://app.circleci.com/settings/project/github/pyutils/line_profiler/environment-variables?return-to=https%3A%2F%2Fapp.circleci.com%2Fpipelines%2Fgithub%2Fpyutils%2Fline_profiler +' + + +setup_package_environs(){ + __doc__=" + Setup environment variables specific for this project. + The remainder of this script should ideally be general to any repo. These + non-secret variables are written to disk and loaded by the script, such + that the specific repo only needs to modify that configuration file. + " + + echo ' + export VARNAME_CI_SECRET="EROTEMIC_CI_SECRET" + export GPG_IDENTIFIER="=Erotemic-CI " + ' | python -c "import sys; from textwrap import dedent; print(dedent(sys.stdin.read()).strip(chr(10)))" > dev/secrets_configuration.sh + + #echo ' + #export VARNAME_CI_SECRET="PYUTILS_CI_SECRET" + #export GPG_IDENTIFIER="=PyUtils-CI " + #' | python -c "import sys; from textwrap import dedent; print(dedent(sys.stdin.read()).strip(chr(10)))" > dev/secrets_configuration.sh +} + + +export_encrypted_code_signing_keys(){ + # You will need to rerun this whenever the signkeys expire and are renewed + + # Load or generate secrets + load_secrets + + source dev/secrets_configuration.sh + + CI_SECRET="${!VARNAME_CI_SECRET}" + echo "CI_SECRET=$CI_SECRET" + echo "GPG_IDENTIFIER=$GPG_IDENTIFIER" + + # ADD RELEVANT VARIABLES TO THE CI SECRET VARIABLES + + # HOW TO ENCRYPT YOUR SECRET GPG KEY + # You need to have a known public gpg key for this to make any sense + + MAIN_GPG_KEYID=$(gpg --list-keys --keyid-format LONG "$GPG_IDENTIFIER" | head -n 2 | tail -n 1 | awk '{print $1}') + GPG_SIGN_SUBKEY=$(gpg --list-keys --with-subkey-fingerprints "$GPG_IDENTIFIER" | grep "\[S\]" -A 1 | tail -n 1 | awk '{print $1}') + echo "MAIN_GPG_KEYID = $MAIN_GPG_KEYID" + echo "GPG_SIGN_SUBKEY = $GPG_SIGN_SUBKEY" + + # Only export the signing secret subkey + # Export plaintext gpg public keys, private sign key, and trust info + mkdir -p dev + gpg --armor --export-options export-backup --export-secret-subkeys "${GPG_SIGN_SUBKEY}!" > dev/ci_secret_gpg_subkeys.pgp + gpg --armor --export ${GPG_SIGN_SUBKEY} > dev/ci_public_gpg_key.pgp + gpg --export-ownertrust > dev/gpg_owner_trust + + # Encrypt gpg keys and trust with CI secret + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_public_gpg_key.pgp > dev/ci_public_gpg_key.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/ci_secret_gpg_subkeys.pgp > dev/ci_secret_gpg_subkeys.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -e -a -in dev/gpg_owner_trust > dev/gpg_owner_trust.enc + echo $MAIN_GPG_KEYID > dev/public_gpg_key + + # Test decrpyt + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --list-packets --verbose + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --list-packets --verbose + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc + cat dev/public_gpg_key + + unload_secrets + + # Look at what we did, clean up, and add it to git + ls dev/*.enc + rm dev/*.pgp + rm dev/gpg_owner_trust + git status + git add dev/*.enc + git add dev/gpg_owner_trust + git add dev/public_gpg_key +} + + +_test_gnu(){ + export GNUPGHOME=$(mktemp -d -t) + ls -al $GNUPGHOME + chmod 700 -R $GNUPGHOME + + source dev/secrets_configuration.sh + + gpg -k + + load_secrets + CI_SECRET="${!VARNAME_CI_SECRET}" + echo "CI_SECRET = $CI_SECRET" + + cat dev/public_gpg_key + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc + + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | gpg --import + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | gpg --import-ownertrust + GLKWS=$CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | gpg --import + + + gpg -k + # | gpg --import + # | gpg --list-packets --verbose +} diff --git a/dev/travis_public_gpg_key.pgp.enc b/dev/travis_public_gpg_key.pgp.enc deleted file mode 100644 index 7fc3cfed..00000000 --- a/dev/travis_public_gpg_key.pgp.enc +++ /dev/null @@ -1,16 +0,0 @@ -U2FsdGVkX18leu1FqCbf5paHmrfkdtHWuu8lmIx5DEVDOYVeJvwJG4DtBGUzTw09 -LNayN8/sUPDHQ//A7mgDz4jVIUr+iTVZ2h2RkydSHG6qVsJHGY6dBg8tSZ/qiKcA -r9YsLJBJZms7beUjCwG9nzTMYmRJuQ+DYjTWd4njCBdH00jEZ80hM02YAuOV1ZqQ -wY0S7VitJiyHQC6rsdNMNlViXy49iAoqCYMJyd7cbgAbj2w3QPxGlAWRcbBJ1EcK -hdbuaXdpmx69hQ+d+w1xYiptZufcDQ6hqq40sEzAlnC2ZhwqA+8GUk6whN+5foF9 -U4D4JTSyvbTNXHePGNwMXC2J9F0momKGEPG5JdJO7NLP/E5VLbtzzoDF2tFUD9W0 -vPndpfj3DxPnRJwWag8i3jHOHkrfeSM4kMNUQNyrg6pEKwvw1kVtMi6ATCvdYESl -3OPvcSlSEzCudsBdNxQ7KnEL9/cUOSKRYR6k6K3YG5Y5dZJe0RTN0aCIg9Rv+Eae -pKl4ieIE6UWa/rEu/xEV7cRklDOGhTDlhbebu8Fpdy9M3ZB/l20/+H7LG5LLT6Cm -7opPPPr+x0go4uKkKwYKYDxv6LvF/wmQqdMQY/X7nE+/n/8ywjlVtjL/ZMRQIAN6 -lupnxJJhTOhHYF89GPD4jDQ+gMIv9NL60mhiRO9+Rc5fG6qozHYNd5C2559P9W92 -tbWSK5nUxfFI0byFc9NNLv1fq7ZdY1dzuFg0P/Q4yjyw7qugdg1Sv7ufNACEm5Rl -Wlh4zh6ii7CXVnge0wo/grGUCskJTfJ/xH16cZFrkSEJP4m4URRkcgD6FXAv+VHH -jMCJNMxIADKCJwBCxnvYaf+8Ii/iX8GP3/GtGjg8nQJo7Zlt1hT5R1xWabnc/+r5 -vxjz7LveUp66bdzoxxMoBFuSYuhIkauCz+sLcG9HYBr/fhtkzlptkWp2Zparn+Iq -Caxtf4A2r8wb2sAbzZycRdsXaQ5aG6pye4uUA69YJka4RYQ1vBEGQ1APmcg6sEHY diff --git a/dev/travis_secret_gpg_key.pgp.enc b/dev/travis_secret_gpg_key.pgp.enc deleted file mode 100644 index ecf0adb0..00000000 --- a/dev/travis_secret_gpg_key.pgp.enc +++ /dev/null @@ -1,18 +0,0 @@ -U2FsdGVkX1+sgU5WO8D5WNNm5U1j12+x1sN0eey2VcZS5MB6t9oTHUcQQVIwQfq5 -pJAd8us+905kSGTOMIS2HlvppZvDI/byuIsh+yPvoPHih8JH6rAYtgRUzfhUn9Ex -3OGSXvj3Hy9YigedJQphtGf/Zg+EAo+uNzHYTvqeEV7GF+ZlGrk/BJmd5+1Do07a -jNK5h5mqLgJ8xL6KoUwQliW1/JWgvfVVEVVaH2tW+QGEvU4tPRudHzT7Adlqc1TP -w1XK+kCMZLrL3smrQL66hWreMEsRkhxZ+3rzFU7pjbmkhMZ84epNUDR/ChiV1Srq -yUxFesZGXta+W+UDV3n7MB3JOBmOrCV1wes3XbpZ/SJbflisjQX6cuSrpu20cW/K -Jv3i1ygAixQjq5Ojno1TihBM38gGvliEIurTqkd3sYj3vgPwXu3KL6KA3IrQjyfp -qMgKYA2DWy3Wau04Ytc4UxP8H4eNV8DdvmCDsIzZA28XGjAiC4DJ2CHCKw0wk3U6 -MLj5zzSxHbXZ02xWRQDXuep7Cy+fzphg5Oh2sjj8MODR9BJGLo+RqSeXPy780nKR -hxiO60FADUNUuXPbabo9EGw7+TPsoMWG4oDoCgGKQGCTyuOxo3CcW64v6mF8skql -CsgXgu/PtySBwtDTOimEszhr2cy74C3YdpTSbTmDOytk1MO9GOgHz0aR1D/j15lQ -T2OllCXDJWzSeD5Fus3WtGuOm8nT9kE3qYxnLjPAPUEq5asyWTpQfJltMyeWUh8U -Inb5Fpj169ZtSPBm0l1L9Kkm9zzlAVBowXD25sekTleqTa/3FOO1GzPkmz84w46S -aCTsnPvY6J68dXR8epCT0Yeop6q9x3QLm/q2rUKrMXQYv3DYGpVyfVltoYcSgk/E -2TjzRRiaDccjbqW0FTd2x/70aoLdD8enJqcYpeOZghqpEcdzBBWDnmGD0N60obmx -9a9++3FmcmeCXv5BZUkDu8+Ljy+nZIn+lFgkr64Z3BDoBmfZ2YJciG33H/z9deCZ -JFL4N/+EDMC/7N0NgiYBPyfWAgmkjo8o7rphCPajYSD5DjRX6DVyyMAWOAE5DcsN -6BQv/Q0GhThcnOcgHvvp2QW0QTmssAKeGhcY6FGHzqqrkBPViohfdWJjWjgbrqkD diff --git a/publish.sh b/publish.sh index 1c9c912e..96eb6901 100755 --- a/publish.sh +++ b/publish.sh @@ -34,6 +34,7 @@ Usage: # Set your variables or load your secrets export TWINE_USERNAME= export TWINE_PASSWORD= + TWINE_REPOSITORY_URL="https://test.pypi.org/legacy/" source $(secret_loader.sh) @@ -107,6 +108,13 @@ DO_TAG=$(normalize_boolean "$DO_TAG") TWINE_USERNAME=${TWINE_USERNAME:=""} TWINE_PASSWORD=${TWINE_PASSWORD:=""} +if [[ "$(cat .git/HEAD)" != "ref: refs/heads/release" ]]; then + # If we are not on release, then default to the test pypi upload repo + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://test.pypi.org/legacy/"} +else + TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL:="https://upload.pypi.org/legacy/"} +fi + if [[ "$(which gpg2)" != "" ]]; then GPG_EXECUTABLE=${GPG_EXECUTABLE:=gpg2} else @@ -121,6 +129,7 @@ echo " === PYPI BUILDING SCRIPT == VERSION='$VERSION' TWINE_USERNAME='$TWINE_USERNAME' +TWINE_REPOSITORY_URL = $TWINE_REPOSITORY_URL GPG_KEYID = '$GPG_KEYID' MB_PYTHON_TAG = '$MB_PYTHON_TAG' @@ -305,9 +314,13 @@ if [[ "$DO_UPLOAD" == "True" ]]; then for WHEEL_PATH in "${WHEEL_PATHS[@]}" do if [ "$DO_GPG" == "True" ]; then - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD --sign $WHEEL_PATH.asc $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD \ + --repository-url $TWINE_REPOSITORY_URL \ + --sign $WHEEL_PATH.asc $WHEEL_PATH --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; } else - twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD $WHEEL_PATH || { echo 'failed to twine upload' ; exit 1; } + twine upload --username $TWINE_USERNAME --password=$TWINE_PASSWORD \ + --repository-url $TWINE_REPOSITORY_URL \ + $WHEEL_PATH --skip-existing --verbose || { echo 'failed to twine upload' ; exit 1; } fi done echo """ diff --git a/pytest.ini b/pytest.ini index ed5ece45..b25ad93c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ # ON COVERAGE OF PYTEST PLUGINS: # http://pytest-cov.readthedocs.io/en/latest/plugins.html addopts = -p pytester -p no:doctest --xdoctest --ignore-glob=setup.py --ignore=.tox --ignore=setup.py --ignore=dev -norecursedirs = .git ignore build __pycache__ docs *.egg-info _* dev testing/pybind11_test setup.py +norecursedirs = .git ignore build __pycache__ docs *.egg-info dev testing/pybind11_test setup.py # --pyargs --doctest-modules --ignore=.tox ;rsyncdirs = tox.ini pytest.py _pytest testing ;python_files = test_*.py *_test.py testing/*/*.py diff --git a/run_tests.py b/run_tests.py index f413ee8e..06df7e5c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,6 +4,7 @@ import pytest import sys package_name = 'xdoctest' + package_dpath = package_name pytest_args = [ '-p', 'pytester', '-p', 'no:doctest', @@ -12,7 +13,7 @@ '--cov-report', 'html', '--cov-report', 'term', '--cov=' + package_name, - package_name, 'testing' + package_dpath, 'testing' ] pytest_args = pytest_args + sys.argv[1:] print('pytest.__version__ = {!r}'.format(pytest.__version__)) diff --git a/testing/test_core.py b/testing/test_core.py index 03b5d522..23cba1e7 100644 --- a/testing/test_core.py +++ b/testing/test_core.py @@ -499,6 +499,106 @@ def test_backwards_compat_indent_value(): assert status['passed'] +def test_concise_try_except(): + """ + CommandLine: + xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_try_except + """ + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> # xdoctest: +IGNORE_WANT + >>> try: raise Exception + ... except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) + a bad want string + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> # xdoctest: +IGNORE_WANT + >>> try: raise Exception + >>> except Exception: print(lambda *a, **b: sys.stdout.write(str(a) + "\n" + str(b))) + a bad want string + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + +def test_semicolon_line(): + r""" + Test for https://github.com/Erotemic/xdoctest/issues/108 + + Note: + Notes on the issue: + + .. code:: python + # This works + compile("import os; print(os)", filename="", mode='exec') + compile("import os; print(os)", filename="", mode='single') + + compile("1; 2", filename="", mode='exec') + compile("1; 2", filename="", mode='single') + + compile("print();print()", filename="", mode='single') + compile("print();print()", filename="", mode='exec') + + compile("print()", filename="", mode='eval') + compile("print()", filename="", mode='exec') + compile("print()", filename="", mode='single') + + # This breaks: + compile("import os; print(os)", filename="", mode='eval') + + # I suppose we can't have imports in an eval? + compile("import os\n", filename="", mode='eval') + + # Or multiple lines? + compile("print();print()", filename="", mode='eval') + + # No imports, no assignments, no semicolons + compile("1; 2", filename="", mode='eval') + + + CommandLine: + xdoctest -m ~/code/xdoctest/testing/test_core.py test_concise_exceptions + """ + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os; print(os.path.abspath('.')) + """)) + status = example.run(verbose=0) + assert status['passed'] + + # The problem case was when it was compiled with a "want" statement + # + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os; print(os.path.abspath('.')) + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + # Test single import + # import xdoctest + # xdoctest.parser.DEBUG = 100 + from xdoctest.doctest_example import DocTest + example = DocTest( + utils.codeblock(r""" + >>> import os + ... + """)) + status = example.run(verbose=0) + assert status['passed'] + + if __name__ == '__main__': """ CommandLine: diff --git a/testing/test_parser.py b/testing/test_parser.py index 2a16a2ac..503d3cdb 100644 --- a/testing/test_parser.py +++ b/testing/test_parser.py @@ -166,7 +166,7 @@ def test_label_indented_lines(): ] if labeled != expected: try: - import ubelt as ub + # import ubelt as ub # NOQA import itertools as it for got, want in it.zip_longest(labeled, expected): if got != want: @@ -191,8 +191,8 @@ def test_ps1_linenos_1(): 1 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 1] @@ -206,8 +206,8 @@ def test_ps1_linenos_2(): x = 21 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 3] @@ -221,8 +221,8 @@ def test_ps1_linenos_3(): 'x = 21' ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert not eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'exec' assert linenos == [0, 3] @@ -253,8 +253,8 @@ def test_ps1_linenos_4(): 59 ''').split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 3, 5, 9, 13, 16, 17, 20] @@ -269,8 +269,8 @@ def test_retain_source(): ''') source_lines = source.split('\n')[:-1] self = parser.DoctestParser() - linenos, eval_final = self._locate_ps1_linenos(source_lines) - assert eval_final + linenos, mode_hint = self._locate_ps1_linenos(source_lines) + assert mode_hint == 'eval' assert linenos == [0, 1] p1, p2 = self.parse(source) assert p1.source == 'x = 2' @@ -333,9 +333,9 @@ def test_parse_eval_nowant(): self = parser.DoctestParser() parts = self.parse(string) raw_source_lines = string.split('\n')[:] - ps1_linenos, eval_final = self._locate_ps1_linenos(raw_source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(raw_source_lines) assert ps1_linenos == [0, 1] - assert eval_final + assert mode_hint == 'eval' # Only one part because there is no want assert len(parts) == 1 @@ -350,9 +350,9 @@ def test_parse_eval_single_want(): self = parser.DoctestParser() parts = self.parse(string) raw_source_lines = string.split('\n')[:-1] - ps1_linenos, eval_final = self._locate_ps1_linenos(raw_source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(raw_source_lines) assert ps1_linenos == [0, 1] - assert eval_final + assert mode_hint == 'eval' # Only one part because there is no want assert len(parts) == 2 @@ -366,7 +366,7 @@ def test_parse_comment(): labeled = self._label_docsrc_lines(string) assert labeled == [('dsrc', '>>> # nothing')] source_lines = string.split('\n')[:] - linenos, eval_final = self._locate_ps1_linenos(source_lines) + linenos, mode_hint = self._locate_ps1_linenos(source_lines) parts = self.parse(string) assert parts[0].source.strip().startswith('#') @@ -489,7 +489,7 @@ def test_repl_twoline(): def test_repl_comment_in_string(): source_lines = ['>>> x = """', ' # comment in a string', ' """'] self = parser.DoctestParser() - assert self._locate_ps1_linenos(source_lines) == ([0], False) + assert self._locate_ps1_linenos(source_lines) == ([0], 'exec') source_lines = [ '>>> x = """', @@ -500,7 +500,7 @@ def test_repl_comment_in_string(): ' """', ] self = parser.DoctestParser() - assert self._locate_ps1_linenos(source_lines) == ([0, 3], False) + assert self._locate_ps1_linenos(source_lines) == ([0, 3], 'exec') def test_inline_directive(): @@ -534,8 +534,6 @@ def test_inline_directive(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] - # print(ps1_linenos) # [0, 1, 3, 4, 7, 8, 10, 11, 12] # assert ps1_linenos == [0, 2, 5, 6, 8, 9, 10] parts = self.parse(string) @@ -563,7 +561,6 @@ def test_block_directive_nowant1(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -591,7 +588,6 @@ def test_block_directive_nowant2(): ''') # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) # TODO: finsh me assert len(parts) == 2 @@ -608,9 +604,7 @@ def test_block_directive_want1_assign(): >>> _ = func2() # assign this line so we dont break it off for eval want ''') - # source_lines = string.splitlines() self = parser.DoctestParser() - # ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -634,9 +628,7 @@ def test_block_directive_want1_eval(): >>> func2() # eval this line so it is broken off want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 2 @@ -653,9 +645,7 @@ def test_block_directive_want2_assign(): >>> _ = func3() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 2 @@ -672,9 +662,7 @@ def test_block_directive_want2_eval(): >>> func3() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) print('----') for part in parts: @@ -705,9 +693,7 @@ def test_block_directive_want2_eval2(): >>> func4() want ''') - source_lines = string.splitlines() self = parser.DoctestParser() - ps1_linenos = self._locate_ps1_linenos(source_lines)[0] parts = self.parse(string) assert len(parts) == 4 diff --git a/testing/test_plugin.py b/testing/test_plugin.py index a08e022d..33aa2a21 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -2,6 +2,7 @@ """ Adapted from the original `pytest/testing/test_doctest.py` module at: https://github.com/pytest-dev/pytest + https://github.com/pytest-dev/pytest/blob/main/testing/test_doctest.py """ from __future__ import print_function, division, absolute_import, unicode_literals import sys @@ -229,20 +230,21 @@ def test_encoding(self, testdir, test_string, encoding): """Test support for xdoctest_encoding ini option. CommandLine: - pytest testing/test_plugin.py::TestXDoctest::test_encoding + pytest testing/test_plugin.py::TestXDoctest::test_encoding -s -v """ testdir.makeini(""" [pytest] xdoctest_encoding={0} """.format(encoding)) - xdoctest = u""" + doctest = u""" >>> u"{0}" {1} """.format(test_string, repr(test_string)) - testdir._makefile(".txt", [xdoctest], {}, encoding=encoding) + print(doctest) + testdir._makefile(".txt", [doctest], {}, encoding=encoding) - result = testdir.runpytest(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '*1 passed*', @@ -257,7 +259,7 @@ def test_encoding(self, pytester, test_string, encoding): pytester.makeini( """ [pytest] - doctest_encoding={} + xdoctest_encoding={} """.format( encoding ) @@ -271,7 +273,7 @@ def test_encoding(self, pytester, test_string, encoding): fn = pytester.path / "test_encoding.txt" fn.write_text(doctest, encoding=encoding) - result = pytester.runpytest() + result = pytester.runpytest("--xdoctest", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(["*1 passed*"]) @@ -535,7 +537,7 @@ def test_doctest_unex_importerror_only_txt(self, testdir): testdir.maketxtfile(""" >>> import asdalsdkjaslkdjasd """) - result = testdir.runpytest(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) # xdoctest is never executed because of error during hello.py collection result.stdout.fnmatch_lines([ "*>>> import asdals*", @@ -627,14 +629,14 @@ def somefunc(): def test_txtfile_failing(self, testdir): """ CommandLine: - pytest testing/test_plugin.py::TestXDoctest::test_txtfile_failing + pytest testing/test_plugin.py::TestXDoctest::test_txtfile_failing -s """ p = testdir.maketxtfile(""" >>> i = 0 >>> i + 1 2 """) - result = testdir.runpytest(p, "-s", *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", "-s", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '*1 >>> i = 0', '*2 >>> i + 1', @@ -657,7 +659,7 @@ def test_txtfile_with_fixtures(self, testdir): >>> type(dir).__name__ 'LocalPath' """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) def test_txtfile_with_usefixtures_in_ini(self, testdir): @@ -681,7 +683,7 @@ def myfixture(monkeypatch): >>> os.environ["HELLO"] 'WORLD' """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) def test_ignored_whitespace(self, testdir): @@ -783,7 +785,7 @@ def test_xdoctest_multiline_list(self, testdir): >>> print(len(x)) 6 """) - result = testdir.runpytest(p, *EXTRA_ARGS) + result = testdir.runpytest(p, "--xdoctest-modules", *EXTRA_ARGS) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_multiline_string(self, testdir): @@ -817,7 +819,7 @@ def test_xdoctest_multiline_string(self, testdir): >>> '''.strip()) Just prefix everything with >>> and the xdoctest should work """).lstrip()) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_trycatch(self, testdir): @@ -848,7 +850,7 @@ def test_xdoctest_trycatch(self, testdir): foo bar """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_xdoctest_functions(self, testdir): @@ -871,7 +873,7 @@ def test_xdoctest_functions(self, testdir): >>> func() now the ast parser makes doctests nice for us """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines(['* 1 passed*']) def test_stdout_capture_no(self, testdir): @@ -1092,7 +1094,7 @@ def test_unicode_string(self, testdir): >>> b'12'.decode('ascii') '12' """) - reprec = testdir.inline_run(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) passed = int(sys.version_info[0] >= 3) reprec.assertoutcome(passed=passed, failed=int(not passed)) @@ -1106,7 +1108,7 @@ def test_bytes_literal(self, testdir): >>> b'foo' 'foo' """) - reprec = testdir.inline_run(*(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run("--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) passed = int(sys.version_info[0] == 2) reprec.assertoutcome(passed=passed, failed=int(not passed)) @@ -1372,7 +1374,7 @@ def add_contextlib(doctest_namespace): >>> print(cl.__name__) contextlib """) - reprec = testdir.inline_run(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + reprec = testdir.inline_run(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) reprec.assertoutcome(passed=1) @pytest.mark.parametrize('scope', SCOPES) @@ -1690,7 +1692,7 @@ def test_unicode_doctest(self, testdir): >>> 1/0 # Byé 1 """) - result = testdir.runpytest(p, *(EXTRA_ARGS + OLD_TEXT_ARGS)) + result = testdir.runpytest(p, "--xdoctest-modules", *(EXTRA_ARGS + OLD_TEXT_ARGS)) result.stdout.fnmatch_lines([ '* REASON: ZeroDivisionError*', '*1 failed*', diff --git a/testing/test_pytest_cli.py b/testing/test_pytest_cli.py index 25c83636..c187d7aa 100644 --- a/testing/test_pytest_cli.py +++ b/testing/test_pytest_cli.py @@ -45,11 +45,8 @@ def test_simple_pytest_import_error_cli(): This test case triggers an excessively long callback in xdoctest < dev/0.15.7 - xdoctest ~/code/xdoctest/testing/test_pytest_cli.py test_simple_pytest_import_error_cli - - import sys, ubelt - sys.path.append(ubelt.expandpath('~/code/xdoctest/testing')) - from test_pytest_cli import * # NOQA + CommandLine: + xdoctest ~/code/xdoctest/testing/test_pytest_cli.py test_simple_pytest_import_error_cli """ module_text = utils.codeblock( ''' @@ -70,12 +67,11 @@ def module_func1(): command = sys.executable + ' -m pytest -v -s --xdoctest-verbose=3 --xdoctest ' + temp_module.dpath print(command) info = cmd(command) + # We patched doctest_example so it no longer outputs this in the traceback + assert 'util_import' not in info['out'] print(info['out']) - - # info = cmd('pytest --xdoctest ' + temp_module.modpath) - # print(info['out']) - - assert info['ret'] == 1 + # Note: flaky changes the return code from 1 to 3, so test non-zero + assert info['ret'] != 0 def test_simple_pytest_syntax_error_cli(): @@ -96,9 +92,11 @@ def module_func1(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest --xdoctest ' + temp_module.dpath) print(info['out']) + assert info['ret'] != 0 info = cmd(sys.executable + ' -m pytest --xdoctest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 def test_simple_pytest_import_error_no_xdoctest(): @@ -114,10 +112,11 @@ def test_this(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 - info = cmd('pytest ' + temp_module.dpath) + info = cmd(sys.executable + ' -m pytest ' + temp_module.dpath) print(info['out']) - # assert info['ret'] == 0 + assert info['ret'] != 0 def test_simple_pytest_syntax_error_no_xdoctest(): @@ -133,7 +132,8 @@ def test_this(): temp_module = util_misc.TempModule(module_text) info = cmd(sys.executable + ' -m pytest ' + temp_module.modpath) print(info['out']) + assert info['ret'] != 0 info = cmd(sys.executable + ' -m pytest ' + temp_module.dpath) print(info['out']) - # assert info['ret'] == 0 + assert info['ret'] != 0 diff --git a/xdoctest/__init__.py b/xdoctest/__init__.py index 0246ef23..e8aa68aa 100644 --- a/xdoctest/__init__.py +++ b/xdoctest/__init__.py @@ -280,7 +280,7 @@ def fib(n): mkinit xdoctest --nomods ''' -__version__ = '0.15.8' +__version__ = '0.15.9' # Expose only select submodules diff --git a/xdoctest/doctest_example.py b/xdoctest/doctest_example.py index 66194b31..628d3d26 100644 --- a/xdoctest/doctest_example.py +++ b/xdoctest/doctest_example.py @@ -609,8 +609,9 @@ def run(self, verbose=None, on_error=None): # part.compile_mode can be single, exec, or eval. # Typically single is used instead of eval self._partfilename = '' + source_text = part.compilable_source() code = compile( - part.source, mode=part.compile_mode, + source_text, mode=part.compile_mode, filename=self._partfilename, flags=compileflags, dont_inherit=True ) @@ -930,7 +931,7 @@ def repr_failure(self, with_tb=True): # lines += ['{}'.format(list(failed_part.directives))] # lines += ['Failed part source:'] - # lines += failed_part.source.splitlines() + # lines += failed_part.exec_lines # lines += ['Failed part want:'] # if failed_part.want_lines: # lines += failed_part.want_lines diff --git a/xdoctest/doctest_part.py b/xdoctest/doctest_part.py index a0fcbeb0..d998820d 100644 --- a/xdoctest/doctest_part.py +++ b/xdoctest/doctest_part.py @@ -63,6 +63,15 @@ def n_want_lines(self): def source(self): return '\n'.join(self.exec_lines) + def compilable_source(self): + """ + Use this to build the string for compile. Takes care of a corner case. + """ + if self.compile_mode == 'single': + return '\n'.join(self.exec_lines + ['']) + else: + return '\n'.join(self.exec_lines) + @property def directives(self): """ diff --git a/xdoctest/parser.py b/xdoctest/parser.py index 9eab6ab0..de69f192 100644 --- a/xdoctest/parser.py +++ b/xdoctest/parser.py @@ -39,6 +39,7 @@ import ast import sys import re +import tokenize from xdoctest import utils from xdoctest import directive from xdoctest import exceptions @@ -48,15 +49,20 @@ DEBUG = '--debug' in sys.argv - INDENT_RE = re.compile(r'^([ ]*)(?=\S)', re.MULTILINE) +# This issue was resolved in 3.8 +NEED_16806_WORKAROUND = sys.version_info[0:2] < (3, 8) +PY2 = (sys.version_info.major == 2) + + class DoctestParser(object): r""" Breaks docstrings into parts using the `parse` method. Example: + >>> from xdoctest.parser import * # NOQA >>> parser = DoctestParser() >>> doctest_parts = parser.parse( >>> ''' @@ -122,6 +128,7 @@ def parse(self, string, info=None): directive is still in effect. Example: + >>> from xdoctest.parser import * # NOQA >>> from xdoctest import parser >>> from xdoctest.docstr import docscrape_google >>> from xdoctest import core @@ -134,6 +141,7 @@ def parse(self, string, info=None): >>> doctest_parts = self.parse(string) >>> # each part with a want-string needs to be broken in two >>> assert len(doctest_parts) == 6 + >>> len(doctest_parts) """ if DEBUG > 1: print('\n===== PARSE ====') @@ -176,9 +184,9 @@ def parse(self, string, info=None): tb_text = ub.indent(tb_text) print(tb_text) - print('Failed to parse string = <{[<{[<{[') + print('Failed to parse string = <{[<{[<{[ # xdoc debug') print(string) - print(']}>a]}>]}> # end string') + print(']}>]}>]}> # xdoc debug end string') print('info = {}'.format(ub.repr2(info))) print('-----') @@ -251,9 +259,9 @@ def _package_chunk(self, raw_source_lines, raw_want_lines, lineno=0): if DEBUG > 1: print(' * locate ps1 lines') # Find the line number of each standalone statement - ps1_linenos, eval_final = self._locate_ps1_linenos(source_lines) + ps1_linenos, mode_hint = self._locate_ps1_linenos(source_lines) if DEBUG > 1: - print('eval_final = {!r}'.format(eval_final)) + print('mode_hint = {!r}'.format(mode_hint)) print(' * located ps1 lines') # Find all directives here: @@ -303,7 +311,7 @@ def slice_example(s1, s2, want_lines=None): example = slice_example(s1, s2) yield example s1 = s2 - if want_lines and eval_final: + if want_lines and mode_hint in {'eval', 'single'}: # Whenever the evaluation of the final line needs to be tested # against want, that line must be separated into its own part. # We break the last line off so we can eval its value, but keep @@ -316,17 +324,17 @@ def slice_example(s1, s2, want_lines=None): s2 = None example = slice_example(s1, s2, want_lines) + + # if mode_hint is False: + # mode_hint = 'exec' + # if mode_hint is True: + # mode_hint = 'eval' + if not bool(want_lines): example.compile_mode = 'exec' else: - if eval_final is True: - example.compile_mode = 'eval' - elif eval_final is False: - example.compile_mode = 'exec' - elif eval_final == 'single': - example.compile_mode = 'single' - else: - raise KeyError(eval_final) + assert mode_hint in {'eval', 'exec', 'single'} + example.compile_mode = mode_hint if DEBUG > 1: print('example.compile_mode = {!r}'.format(example.compile_mode)) @@ -435,23 +443,48 @@ def _locate_ps1_linenos(self, source_lines): Returns: Tuple[List[int], bool]: - a list of indices indicating which lines are considered "PS1" - and a flag indicating if the final line should be considered - for a got/want assertion. + linenos is the first value a list of indices indicating which + lines are considered "PS1" and + mode_hint, the second value, is a flag indicating if the final + line should be considered for a got/want assertion. Example: >>> self = DoctestParser() >>> source_lines = ['>>> def foo():', '>>> return 0', '>>> 3'] - >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 2] - >>> assert eval_final is True + >>> assert mode_hint == 'eval' Example: >>> self = DoctestParser() >>> source_lines = ['>>> x = [1, 2, ', '>>> 3, 4]', '>>> print(len(x))'] - >>> linenos, eval_final = self._locate_ps1_linenos(source_lines) + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) >>> assert linenos == [0, 2] - >>> assert eval_final is True + >>> assert mode_hint == 'eval' + + Example: + >>> from xdoctest.parser import * # NOQA + >>> self = DoctestParser() + >>> source_lines = [ + >>> '>>> x = 1', + >>> '>>> try: raise Exception', + >>> '>>> except Exception: pass', + >>> '...', + >>> ] + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) + >>> assert linenos == [0, 1] + >>> assert mode_hint == 'exec' + + Example: + >>> from xdoctest.parser import * # NOQA + >>> self = DoctestParser() + >>> source_lines = [ + >>> '>>> import os; print(os)', + >>> '...', + >>> ] + >>> linenos, mode_hint = self._locate_ps1_linenos(source_lines) + >>> assert linenos == [0] + >>> assert mode_hint == 'single' """ # Strip indentation (and PS1 / PS2 from source) exec_source_lines = [p[4:] for p in source_lines] @@ -511,9 +544,12 @@ def balanced_intervals(lines): syn_ex.text = line + '\n' raise syn_ex + # print(ast.dump(pt)) + # print('pt = {!r}'.format(pt)) + statement_nodes = pt.body ps1_linenos = [node.lineno - 1 for node in statement_nodes] - NEED_16806_WORKAROUND = True + # NEED_16806_WORKAROUND = 1 if NEED_16806_WORKAROUND: # pragma: nobranch ps1_linenos = self._workaround_16806( ps1_linenos, exec_source_lines) @@ -524,17 +560,25 @@ def balanced_intervals(lines): } ps1_linenos = sorted(set(ps1_linenos).difference(ps2_linenos)) + # There are 3 ways to compile python code + # exec, eval, and single. + + # We almost always want to exec, but if we want to match the return + # value of the function, we will need to run it in eval or single mode. + mode_hint = 'exec' if len(statement_nodes) == 0: - eval_final = False + mode_hint = 'exec' else: # Is the last statement evaluate-able? - if sys.version_info.major == 2: # nocover - eval_final = isinstance(statement_nodes[-1], ( - ast.Expr, ast.Print)) + if PY2: # nocover + # Python 2 overhead + if isinstance(statement_nodes[-1], (ast.Expr, ast.Print)): + mode_hint = 'eval' else: - # This should just be an Expr in python3 - # (todo: ensure this is true) - eval_final = isinstance(statement_nodes[-1], ast.Expr) + if isinstance(statement_nodes[-1], ast.Expr): + # This should just be an Expr in python3 + # (todo: ensure this is true) + mode_hint = 'eval' # WORKON_BACKWARDS_COMPAT_CONTINUE_EVAL: # Force doctests parts to evaluate in backwards compatible "single" @@ -542,9 +586,28 @@ def balanced_intervals(lines): if len(source_lines) > 1: if source_lines[0].startswith('>>> '): if all(_hasprefix(s, ('...',)) for s in source_lines[1:]): - eval_final = 'single' + mode_hint = 'single' + + if mode_hint == 'eval': + # Also check the tokens in the source lines to look for semicolons + # to fix #108 + # Only iterate through non-empty lines otherwise tokenize will stop short + # TODO: we probably could just save the tokens if we got them earlier? + iterable = (line for line in exec_source_lines if line) + def _readline(): + return next(iterable) + # We cannot eval a statement with a semicolon in it + # Single should work. + if PY2: + if any(t[0] == tokenize.OP and t[1] == ';' + for t in tokenize.generate_tokens(_readline)): + mode_hint = 'single' + else: + if any(t.type == tokenize.OP and t.string == ';' + for t in tokenize.generate_tokens(_readline)): + mode_hint = 'single' - return ps1_linenos, eval_final + return ps1_linenos, mode_hint @staticmethod def _workaround_16806(ps1_linenos, exec_source_lines): @@ -664,6 +727,12 @@ def _label_docsrc_lines(self, string): for line_idx, line in line_iter: match = INDENT_RE.search(line) line_indent = 0 if match is None else (match.end() - match.start()) + if DEBUG: # nocover + print('Next line {}: {}'.format(line_idx, line)) + print('state_indent = {!r}'.format(state_indent)) + print('match = {!r}'.format(match)) + print('line_indent = {!r}'.format(line_indent)) + norm_line = line[state_indent:] # Normalize line indentation strip_line = line.strip() @@ -731,6 +800,7 @@ def _label_docsrc_lines(self, string): print('completing source') for part, norm_line in _complete_source(line, state_indent, line_iter): if DEBUG > 4: # nocover + print('Append Completion Line:') print('part = {!r}'.format(part)) print('norm_line = {!r}'.format(norm_line)) print('curr_state = {!r}'.format(curr_state)) @@ -783,11 +853,25 @@ def _complete_source(line, state_indent, line_iter): """ helper remove lines from the iterator if they are needed to complete source + + This uses :func:`static.is_balanced_statement` to do the heavy lifting + + Example: + >>> from xdoctest.parser import * # NOQA + >>> from xdoctest.parser import _complete_source + >>> state_indent = 0 + >>> line = '>>> x = { # The line is not finished' + >>> remain_lines = ['>>> 1:2,', '>>> 3:4,', '>>> 5:6}', '>>> y = 7'] + >>> line_iter = enumerate(remain_lines, start=1) + >>> finished = list(_complete_source(line, state_indent, line_iter)) + >>> final = chr(10).join([t[1] for t in finished]) + >>> print(final) """ norm_line = line[state_indent:] # Normalize line indentation prefix = norm_line[:4] suffix = norm_line[4:] - assert prefix.strip() in {'>>>', '...'}, '{}'.format(prefix) + assert prefix.strip() in {'>>>', '...'}, ( + 'unexpected prefix: {!r}'.format(prefix)) yield line, norm_line source_parts = [suffix] diff --git a/xdoctest/static_analysis.py b/xdoctest/static_analysis.py index 80b53740..0b1297e9 100644 --- a/xdoctest/static_analysis.py +++ b/xdoctest/static_analysis.py @@ -841,7 +841,7 @@ def package_modpaths(pkgpath, with_pkg=False, with_mod=True, followlinks=True, break -def is_balanced_statement(lines, only_tokens=False): +def is_balanced_statement(lines, only_tokens=False, reraise=0): r""" Checks if the lines have balanced braces and quotes. @@ -900,6 +900,13 @@ def is_balanced_statement(lines, only_tokens=False): >>> ] >>> print('\n'.join(source_parts)) >>> assert is_balanced_statement(source_parts) + + Doctest: + >>> lines = ['try: raise Exception'] + >>> is_balanced_statement(lines, only_tokens=1) + True + >>> is_balanced_statement(lines, only_tokens=0) + False """ # Only iterate through non-empty lines otherwise tokenize will stop short lines = list(lines) @@ -912,11 +919,15 @@ def _readline(): except tokenize.TokenError as ex: message = ex.args[0] if message.startswith('EOF in multi-line'): + if reraise: + raise return False raise except IndentationError as ex: message = ex.args[0] if message.startswith('unindent does not match any outer indentation'): + if reraise: + raise return False raise else: @@ -932,8 +943,9 @@ def _readline(): # text = dedent(text) six_axt_parse(text) except SyntaxError: + if reraise: + raise return False - return True