[Resolves #894] Switch to using poetry (#1323)

The intent here is to change the tooling over to use [poetry](https://python-poetry.org) instead of `setuptools`. This is building on the work that was started by @tarkatronic in PR #1035 Some important things to note: * Poetry will be used for packaging, dependencies, and publishing. * Poetry is smart enough to automatically include data files such as the `*.json` that had to be manually included by setuptools. * `twine` is no longer necessary as we can simply use `poetry publish`. * Poetry handles all virtualenv matters automatically. So we no longer need to instruct users to construct virtualenvs for development purposes, nor do we need to manually create and manage them for CI. * Dependency specifications are a bit different with Poetry, and a bit more "modern", [as seen here](https://python-poetry.org/docs/dependency-specification/). * I have told git (via the `.gitattributes`) file to treat the `poetry.lock` file as binary. This is a fairly standard practice, also followed in the JS community with the `package-lock.json` file. This does two primary things: * Tells GitHub to not attempt to display a diff for this file * Prevents frequent merge conflicts, since this tends to be a frequently updated file
Sceptre · Apr 20, 2023 · ac9e439 · ac9e439
1 parent 0058d71
commit ac9e439
Show file tree

Hide file tree

Showing 18 changed files with 2,019 additions and 377 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -18,6 +18,9 @@ aliases:
           REPOSITORY_PATH: '/home/circleci/docs'
           DEPLOYMENT_GIT_SSH: 'git@github.com:Sceptre/sceptre.github.io.git'
     steps:
+      - checkout
+      - restore_cache:
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - add_ssh_keys:
           fingerprints:
             - '80:24:c0:40:28:36:f1:80:7f:fd:15:c3:09:cf:86:fc'
@@ -27,82 +30,58 @@ aliases:
           name: 'Build and deploy docs'
           command: |
             chmod +x .circleci/add-known-hosts.sh
-            ./.circleci/add-known-hosts.sh
-            pyenv global venv
+            poetry run ./.circleci/add-known-hosts.sh
             chmod +x .circleci/github-pages.sh
-            ./.circleci/github-pages.sh
+            poetry run ./.circleci/github-pages.sh
 
 jobs:
   build:
     docker:
       - image: sceptreorg/sceptre-circleci:2.0.0
     steps:
       - checkout
-      - run:
-          name: 'Creating Virtualenv'
-          command: |
-            pyenv virtualenv 3.9.4 venv
       - restore_cache:
-          key:
-            sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-            checksum "requirements/prod.txt" }}-{{ checksum
-            "requirements/dev.txt" }}
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - run:
           name: 'Installing Dependencies'
-          command: |
-            pyenv global venv
-            make install-dev
-      - save_cache:
-          key:
-            sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-            checksum "requirements/prod.txt" }}-{{ checksum
-            "requirements/dev.txt" }}
-          paths:
-            - ../.pyenv/versions/3.9.4/envs/venv
+          command: poetry install --all-extras -v
       - run:
-          name: 'Installing Sceptre'
-          command: |
-            pyenv global venv
-            pip install .
-      - persist_to_workspace:
-          root: /home/circleci
+          name: Run pre-commit
+          command: poetry run pre-commit run --all-files --show-diff-on-failure
+      - run:
+          name: Build package
+          command: poetry build
+      - save_cache:
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
           paths:
-            - project
-            - .pyenv
+            - '~/.cache/pypoetry'
+            - '~/.cache/pre-commit'
+            - '.tox'
 
-  lint-and-unit-tests:
+  unit-tests:
     docker:
       - image: sceptreorg/sceptre-circleci:2.0.0
     steps:
-      - attach_workspace:
-          at: /home/circleci
+      - checkout
       - restore_cache:
-          keys:
-            - sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-              checksum "requirements/prod.txt" }}-{{ checksum "requirements/dev.txt" }}
-            - sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-              checksum "requirements/prod.txt" }}-{{ checksum
-              "requirements/dev.txt" }}-{{ checksum "tox.ini" }}
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - run:
-          name: 'Linting'
+          name: Unit Tests
           command: |
-            pyenv global venv
-            make lint
-      - run:
-          name: 'Unit Test'
-          command: |
-            pyenv global venv
-            make test-all
-      - save_cache:
-          key:
-            sceptre-{{ .Environment.CACHE_VERSION }}-{{ arch
-            }}-{{ checksum "requirements/prod.txt" }}-{{ checksum
-            "requirements/dev.txt" }}-{{ checksum "tox.ini" }}
-          paths:
-            - .tox
+            poetry run tox -e clean
+            poetry run tox
+            poetry run tox -e report
       - store_test_results:
-          path: coverage.xml
+          path: test-reports
+      - store_artifacts:
+          path: htmlcov
           destination: coverage-reports
+      - save_cache:
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
+          paths:
+            - '~/.cache/pypoetry'
+            - '~/.cache/pre-commit'
+            - '.tox'
 
   integration-tests:
     parallelism: 2
@@ -111,23 +90,19 @@ jobs:
         environment:
           AWS_DEFAULT_REGION: eu-west-1
     steps:
-      - attach_workspace:
-          at: /home/circleci
+      - checkout
       - restore_cache:
-          key:
-            sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-            checksum "requirements/prod.txt" }}-{{ checksum
-            "requirements/dev.txt" }}
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - run:
           name: 'Integration Testing'
           command: |
-            pyenv global venv
             mkdir -p ~/.aws
             echo -e "[default]\nregion=eu-west-1\nsource_profile=default\nrole_arn=arn:aws:iam::743644221192:role/sceptre-integration-test-ServiceRole-1SHK9LY0T6P3F" > ~/.aws/config
             echo -e "[default]\nregion=eu-west-1\naws_access_key_id=$CI_SERVICE_AWS_ACCESS_KEY_ID\naws_secret_access_key=$CI_SERVICE_AWS_SECRET_ACCESS_KEY" > ~/.aws/credentials
-            behave --junit \
-                   --junit-directory build/behave \
-                   $(circleci tests glob "integration-tests/features/*.feature" | circleci tests split --split-by=timings)
+            poetry run behave \
+                      --junit \
+                      --junit-directory build/behave \
+                      $(circleci tests glob "integration-tests/features/*.feature" | circleci tests split --split-by=timings)
       - store_test_results:
           path: build/behave
           destination: build/behave
@@ -156,32 +131,29 @@ jobs:
 
   deploy-docs-tag: *docs-job
 
-  deploy-pypi:
+  deploy-pypi-test:
     docker:
       - image: sceptreorg/sceptre-circleci:2.0.0
     steps:
-      - attach_workspace:
-          at: /home/circleci
+      - checkout
       - restore_cache:
-          key:
-            sceptre-{{ .Environment.CACHE_VERSION }}-dependencies-{{ arch }}-{{
-            checksum "requirements/prod.txt" }}-{{ checksum
-            "requirements/dev.txt" }}
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - run:
-          name: 'Installing Dependencies'
+          name: Test upload to PyPi
           command: |
-            pyenv global venv
-            make install-dev
-      - run:
-          name: 'Create Distributions'
-          command: |
-            pyenv global venv
-            make dist
+            poetry config repositories.test-pypi "https://test.pypi.org/legacy/"
+            poetry publish --build -r test-pypi -u $PYPI_TEST_USER -p $PYPI_TEST_PASSWORD
+
+  deploy-pypi-prod:
+    docker:
+      - image: sceptreorg/sceptre-circleci:2.0.0
+    steps:
+      - checkout
+      - restore_cache:
+          key: sceptre-v1-{{ checksum "pyproject.toml" }}-{{ checksum "poetry.lock" }}-{{ checksum ".pre-commit-config.yaml" }}-{{ checksum "tox.ini" }}
       - run:
-          name: 'Upload Distributions'
-          command: |
-            pyenv global venv
-            twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD dist/*
+          name: Upload to PyPi
+          command: poetry publish --build -u $PYPI_PROD_USER -p $PYPI_PROD_PASSWORD
 
   deploy-latest-dockerhub:
     executor: docker-publisher
@@ -237,15 +209,15 @@ workflows:
           filters:
             branches:
               only: /^pull\/.*/
-      - lint-and-unit-tests:
+      - unit-tests:
           requires:
             - build
           filters:
             branches:
               only: /^pull\/.*/
       - build-docker-image:
           requires:
-            - lint-and-unit-tests
+            - build
           filters:
             branches:
               only: /^pull\/.*/
@@ -256,7 +228,7 @@ workflows:
           filters:
             branches:
               ignore: /^pull\/.*/
-      - lint-and-unit-tests:
+      - unit-tests:
           requires:
             - build
           filters:
@@ -269,41 +241,42 @@ workflows:
           filters:
             branches:
               ignore: /^pull\/.*/
+      - deploy-pypi-test:
+          context: sceptre-core
+          requires:
+            - unit-tests
+          filters:
+            tags:
+              only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
+            branches:
+              ignore: /.*/
       - deploy-docs-branch:
           context: sceptre-core
           requires:
-            - lint-and-unit-tests
+            - unit-tests
             - integration-tests
           filters:
             branches:
               ignore: /^pull\/.*/
       - build-docker-image:
           requires:
-            - lint-and-unit-tests
-            - integration-tests
+            - unit-tests
           filters:
             branches:
               ignore: /^pull\/.*/
       - deploy-latest-dockerhub:
           context: sceptreorg-dockerhub
           requires:
+            - integration-tests
             - build-docker-image
           filters:
             branches:
               ignore: /^pull\/.*/
 
   publish:
     jobs:
-      - build:
-          filters:
-            tags:
-              only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
-            branches:
-              ignore: /.*/
-      - deploy-pypi:
+      - deploy-pypi-prod:
           context: sceptre-core
-          requires:
-            - build
           filters:
             tags:
               only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
@@ -312,15 +285,15 @@ workflows:
       - deploy-docs-tag:
           context: sceptre-core
           requires:
-            - deploy-pypi
+            - deploy-pypi-prod
           filters:
             tags:
               only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
             branches:
               ignore: /.*/
       - build-docker-image:
           requires:
-            - deploy-pypi
+            - deploy-pypi-prod
           filters:
             tags:
               only: /^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$/
@@ -338,6 +311,8 @@ workflows:
       - tweet-release:
           context: sceptreorg-twitter
           requires:
+            - deploy-pypi-prod
+            - deploy-docs-tag
             - deploy-dockerhub-tagged
           filters:
             tags:

diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1 @@
+poetry.lock binary
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -6,9 +6,9 @@
 - [ ] Commit message starts with `[Resolve #issue-number]`.
 - [ ] Added/Updated unit tests.
 - [ ] Added/Updated integration tests (if applicable).
-- [ ] All unit tests (`make test`) are passing.
+- [ ] All unit tests (`poetry run tox`) are passing.
 - [ ] Used the same coding conventions as the rest of the project.
-- [ ] The new code passes pre-commit validations (`pre-commit run --all-files`).
+- [ ] The new code passes pre-commit validations (`poetry run pre-commit run --all-files`).
 - [ ] The PR relates to _only_ one subject with a clear title.
       and description in grammatically correct, complete sentences.
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -44,6 +44,13 @@ repos:
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.10
+  - repo: https://github.com/python-poetry/poetry
+    rev: '1.4.0'
+    hooks:
+      - id: poetry-check
+      - id: poetry-lock
+        language_version: python3.10
+        args: ['--check']
   - repo: https://github.com/AleksaC/circleci-cli-py
     rev: v0.1.25638
     hooks: