From 57b74a440f48261b7827b738f1d567b29a3b8f91 Mon Sep 17 00:00:00 2001 From: Moshe Zadka Date: Fri, 6 Jul 2018 16:18:20 -0700 Subject: [PATCH 01/18] Circle 1->2 (#12) * Circle 2.0 configuration * Remove Circle 1.0 configuration * Use autogenerated * Remove prev link that is unretrievable in test --- .circleci/config.yml | 171 ++++++++++++++++++ .globality/build.json | 6 + MANIFEST.in | 0 circle.yml | 35 ---- .../tests/endpoints/test_http_endpoint.py | 3 - 5 files changed, 177 insertions(+), 38 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .globality/build.json create mode 100644 MANIFEST.in delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..2eec606 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,171 @@ +# +# Globality autogenerated CircleCI configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-buid +# +# + +defaults: &defaults + working_directory: ~/repo + docker: + - image: ${AWS_ECR_DOMAIN}/globality-build:stable + aws_auth: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + environment: + EXTRA_INDEX_URL: "InjectedDuringRuntime" + AWS_ECR_DOMAIN: "InjectedDuringRuntime" + JFROG_AUTH: "InjectedDuringRuntime" + PYPI_USERNAME: "InjectedDuringRuntime" + PYPI_PASSWORD: "InjectedDuringRuntime" + +deploy_defaults: &deploy_defaults + working_directory: ~/repo + docker: + - image: ${AWS_ECR_DOMAIN}/globality-build:stable + aws_auth: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + environment: + EXTRA_INDEX_URL: "InjectedDuringRuntime" + AWS_ECR_DOMAIN: "InjectedDuringRuntime" + JFROG_AUTH: "InjectedDuringRuntime" + PYPI_USERNAME: "InjectedDuringRuntime" + PYPI_PASSWORD: "InjectedDuringRuntime" + +whitelist: &whitelist + paths: + . + +version: 2 + +jobs: + checkout: + <<: *defaults + + steps: + - checkout + + - persist_to_workspace: + root: ~/repo + <<: *whitelist + + build_base_docker: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + + - run: + name: Build Base Docker + # install dependencies for loading ecs task definitions + command: | + eval $(aws ecr get-login --no-include-email) + python -m middlefield globality build build-gen --config .globality/build.json + python -m middlefield globality build docker-base --repo microcosm-resourcesync + python -m middlefield globality build docker --repo microcosm-resourcesync + + test: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + - run: + name: Test code + command: | + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + eval $(aws ecr get-login --no-include-email) + docker pull ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} test + + lint: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + + - run: + name: Run Lint + command: | + docker create -v /src/microcosm-resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + eval $(aws ecr get-login --no-include-email) + docker pull ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} lint + + + deploy: + <<: *defaults + steps: + - attach_workspace: + at: ~/repo + - run: + name: Deploy + command: | + echo "[distutils]" > ~/.pypirc + echo "index-servers =" >> ~/.pypirc + echo " pypi" >> ~/.pypirc + echo >> ~/.pypirc + echo "[pypi]" >> ~/.pypirc + echo "username:$PYPI_USERNAME" >> ~/.pypirc + echo "password:$PYPI_PASSWORD" >> ~/.pypirc + echo >> ~/.pypirc + python setup.py register -r pypi + python setup.py sdist upload -r pypi + +workflows: + version: 2 + + build-and-release: + jobs: + - checkout: + filters: + # run for all branches and tags + tags: + only: /.*/ + - build_base_docker: + requires: + - checkout + filters: + # run for all branches and tags + tags: + only: /.*/ + - lint: + requires: + - build_base_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - test: + requires: + - build_base_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - deploy: + requires: + - test + - lint + filters: + branches: + ignore: /.*/ + tags: + only: /^[0-9]+(\.[0-9]+)*/ diff --git a/.globality/build.json b/.globality/build.json new file mode 100644 index 0000000..cddebb4 --- /dev/null +++ b/.globality/build.json @@ -0,0 +1,6 @@ +{ + "type": "python-library", + "params": { + "name": "microcosm-resourcesync" + } +} diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e69de29 diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 1bff0ee..0000000 --- a/circle.yml +++ /dev/null @@ -1,35 +0,0 @@ -# CircleCI Configuration file - -machine: - python: - version: 3.6.2 - -general: - artifacts: - - "dist" - - "cover" - -dependencies: - override: - - pip install tox tox-pyenv - - pyenv local 3.6.2 - -test: - override: - - tox - -deployment: - pypi: - tag: /[0-9]+(\.[0-9]+)*/ - owner: globality-corp - commands: - - echo "[distutils]" > ~/.pypirc - - echo "index-servers =" >> ~/.pypirc - - echo " pypi" >> ~/.pypirc - - echo >> ~/.pypirc - - echo "[pypi]" >> ~/.pypirc - - echo "username:$PYPI_USERNAME" >> ~/.pypirc - - echo "password:$PYPI_PASSWORD" >> ~/.pypirc - - echo >> ~/.pypirc - - python setup.py register -r pypi - - python setup.py sdist upload -r pypi diff --git a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py index 4f69c4a..3e2df82 100644 --- a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py +++ b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py @@ -143,9 +143,6 @@ def test_read_follow_page(self): self=dict( href="http://example.com/api/foo?offset=2&limit2", ), - prev=dict( - href="http://example.com/api/foo?offset=0&limit=2", - ), ), items=[ hal_resource("http://example.com/api/foo/3"), From 5a0fb6bc6842aab20a9bf011c3a7b894bbc984f6 Mon Sep 17 00:00:00 2001 From: Moshe Zadka Date: Mon, 9 Jul 2018 14:05:09 -0700 Subject: [PATCH 02/18] GLOB-21814: Use same repo for all open source libraries (#13) --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2eec606..e484b1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,8 +71,8 @@ jobs: command: | eval $(aws ecr get-login --no-include-email) python -m middlefield globality build build-gen --config .globality/build.json - python -m middlefield globality build docker-base --repo microcosm-resourcesync - python -m middlefield globality build docker --repo microcosm-resourcesync + python -m middlefield globality build docker-base --repo python-library + python -m middlefield globality build docker --repo python-library test: <<: *defaults @@ -88,8 +88,8 @@ jobs: docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ eval $(aws ecr get-login --no-include-email) - docker pull ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} - docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} test + docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} test lint: <<: *defaults @@ -106,8 +106,8 @@ jobs: docker create -v /src/microcosm-resourcesync/tests/ --name service_tests alpine:3.4 /bin/true docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ eval $(aws ecr get-login --no-include-email) - docker pull ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} - docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/microcosm-resourcesync:${CIRCLE_SHA1} lint + docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} lint deploy: From e18112d78bf00e2efd2e7c17958343d6dd312e64 Mon Sep 17 00:00:00 2001 From: Jesse Myers Date: Fri, 19 Oct 2018 12:52:58 -0700 Subject: [PATCH 03/18] Declare python_requires --- microcosm_resourcesync/tests/endpoints/test_http_endpoint.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py index 3e2df82..fc75fe4 100644 --- a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py +++ b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py @@ -3,6 +3,7 @@ """ from json import dumps +from unittest.mock import patch, Mock from hamcrest import ( assert_that, @@ -10,7 +11,6 @@ has_length, is_, ) -from mock import patch, Mock from microcosm_resourcesync.endpoints import HTTPEndpoint from microcosm_resourcesync.following import FollowMode diff --git a/setup.py b/setup.py index f92935b..9af5a26 100755 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), include_package_data=True, zip_safe=False, + python_requires=">=3.6", keywords="microcosm", install_requires=[ "click>=6.7", @@ -33,7 +34,6 @@ }, tests_require=[ "coverage>=4.3.4", - "mock>=2.0.0", "PyHamcrest>=1.9.0", ], ) From e91775f477d476a8888c38b8a91c04b5b7dbd463 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Mon, 17 Dec 2018 17:06:12 -0800 Subject: [PATCH 04/18] Generating build files through globality-build --- .dockerignore | 13 ++++ Dockerfile.template | 39 ++++++++++++ .../docker-base/Dockerfile.deps | 36 +++++++++++ .../docker-base/Dockerfile.template | 60 +++++++++++++++++++ entrypoint.sh | 39 ++++++++++++ 5 files changed, 187 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile.template create mode 100644 build.python-library/docker-base/Dockerfile.deps create mode 100644 build.python-library/docker-base/Dockerfile.template create mode 100755 entrypoint.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..e2582fd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +* +!microcosm_resourcesync +microcosm_resourcesync/test* +!dist/*-none-any.whl +!entrypoint.sh +!MANIFEST.in +!README.md +!requirements.txt +!setup.py +!setup.cfg +**/*.pyc +**/*~ + diff --git a/Dockerfile.template b/Dockerfile.template new file mode 100644 index 0000000..676df9d --- /dev/null +++ b/Dockerfile.template @@ -0,0 +1,39 @@ +# +# Globality autogenerated Docker configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + + +FROM /python-library:base- + +# Setup invocation +# +# We expose the application on the standard HTTP port and use an entrypoint +# to customize the `dev` and `test` targets. + +ENV NAME microcosm_resourcesync +COPY README.md entrypoint.sh /src/ +ENTRYPOINT ["./entrypoint.sh"] + +# Install source +# +# We should not need to reinstall dependencies here, but we do need to import +# the distribution properly. We also save build arguments to the image using +# microcosm-compatible environment variables. + + +ARG BUILD_NUM +ARG SHA1 +ENV MICROCOSM-RESOURCESYNC__BUILD_INFO_CONVENTION__BUILD_NUM ${BUILD_NUM} +ENV MICROCOSM-RESOURCESYNC__BUILD_INFO_CONVENTION__SHA1 ${SHA1} +COPY $NAME /src/$NAME/ +RUN pip install --extra-index-url $EXTRA_INDEX_URL -e . + diff --git a/build.python-library/docker-base/Dockerfile.deps b/build.python-library/docker-base/Dockerfile.deps new file mode 100644 index 0000000..3e7a78f --- /dev/null +++ b/build.python-library/docker-base/Dockerfile.deps @@ -0,0 +1,36 @@ +# +# Globality autogenerated Docker configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + + +# Install from Debian Stretch with modern Python support +FROM python:slim-stretch + +# +# Most services will use the same set of packages here, though a few will install +# custom dependencies for native requirements. +# + +ARG EXTRA_INDEX_URL +ENV EXTRA_INDEX_URL ${EXTRA_INDEX_URL} + +ENV CORE_PACKAGES locales +ENV BUILD_PACKAGES build-essential libffi-dev +ENV OTHER_PACKAGES libssl-dev + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ${CORE_PACKAGES} ${BUILD_PACKAGES} && \ + apt-get install -y --no-install-recommends ${OTHER_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + diff --git a/build.python-library/docker-base/Dockerfile.template b/build.python-library/docker-base/Dockerfile.template new file mode 100644 index 0000000..3469d92 --- /dev/null +++ b/build.python-library/docker-base/Dockerfile.template @@ -0,0 +1,60 @@ +# +# Globality autogenerated Docker configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + +FROM + +# Install dependencies +# +# Since many Python distributions require a compile for their native dependencies +# we install a compiler and any required development libraries before the installation +# and then *remove* the the compiler when we are done. +# +# We can control dependency freezing by managing the contents of `requirements.txt`. +# +# We can speed up the installation a little bit by breaking out the common +# pip dependencies into their own layer, but avoid this optimization for +# now to improve clarity. +# +# We also install the web application server (which should not be one of our +# explicit dependencies). +# +# Many services will need to modify this step for Python libraries with other +# native dependencies. + + +# Work in /src +# +# We'll copy local source code here for development. +WORKDIR src + +# Set a proper locale +# +# UTF-8 everywhere. + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen "en_US.UTF-8" && \ + /usr/sbin/update-locale LANG=en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Install top-level files +# +# These are enough to install dependencies and have a stable base layer +# when source code changes. + +COPY MANIFEST.in setup.cfg setup.py /src/ + +RUN pip install --upgrade --extra-index-url ${EXTRA_INDEX_URL} /src/ && \ + apt-get remove --purge -y ${BUILD_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..2b522a8 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,39 @@ +#!/bin/bash -e + +# Container entrypoint to simplify running the production and dev servers. + +# Entrypoint conventions are as follows: +# +# - If the container is run without a custom CMD, the service should run as it would in production. +# +# - If the container is run with the "dev" CMD, the service should run in development mode. +# +# Normally, this means that if the user's source has been mounted as a volume, the server will +# restart on code changes and will inject extra debugging/diagnostics. +# +# - If the CMD is "test" or "lint", the service should run its unit tests or linting. +# +# There is no requirement that unit tests work unless user source has been mounted as a volume; +# test code should not normally be shipped with production images. +# +# - Otherwise, the CMD should be run verbatim. + + +if [ "$1" = "test" ]; then + # Install standard test dependencies; YMMV + pip --quiet install \ + .[test] nose mock PyHamcrest coverage + exec nosetests ${NAME} +elif [ "$1" = "lint" ]; then + # Install standard linting dependencies; YMMV + pip --quiet install \ + .[lint] flake8 flake8-print flake8-logging-format + exec flake8 ${NAME} +elif [ "$1" = "typehinting" ]; then + # Install standard type-linting dependencies + pip --quiet install mypy + exec mypy ${NAME} --ignore-missing-imports +else + echo "Cannot execute $@" + exit 3 +fi From 98dfa51b933da5f53d2be07ba76084c724e9d437 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Mon, 17 Dec 2018 17:06:15 -0800 Subject: [PATCH 05/18] Generating build files through globality-build --- .globality/build.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.globality/build.json b/.globality/build.json index cddebb4..b550b6c 100644 --- a/.globality/build.json +++ b/.globality/build.json @@ -1,6 +1,7 @@ { - "type": "python-library", "params": { "name": "microcosm-resourcesync" - } + }, + "type": "python-library", + "version": "2.0.36" } From 24fe26ef3a50825cc931895d848d958a879f9e50 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Mon, 17 Dec 2018 17:06:36 -0800 Subject: [PATCH 06/18] Generating build files through globality-build --- .circleci/config.yml | 47 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e484b1c..31b0049 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,14 +7,14 @@ # Any changes made to this file will be overwritten in the # next version of the build. # -# See: http://github.com/globality-corp/globality-buid +# See: http://github.com/globality-corp/globality-build # # defaults: &defaults working_directory: ~/repo docker: - - image: ${AWS_ECR_DOMAIN}/globality-build:stable + - image: ${AWS_ECR_DOMAIN}/globality-build:2.0.0 aws_auth: aws_access_key_id: ${AWS_ACCESS_KEY_ID} aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} @@ -28,7 +28,7 @@ defaults: &defaults deploy_defaults: &deploy_defaults working_directory: ~/repo docker: - - image: ${AWS_ECR_DOMAIN}/globality-build:stable + - image: ${AWS_ECR_DOMAIN}/globality-build:2.0.0 aws_auth: aws_access_key_id: ${AWS_ACCESS_KEY_ID} aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} @@ -70,9 +70,8 @@ jobs: # install dependencies for loading ecs task definitions command: | eval $(aws ecr get-login --no-include-email) - python -m middlefield globality build build-gen --config .globality/build.json - python -m middlefield globality build docker-base --repo python-library - python -m middlefield globality build docker --repo python-library + globality-build docker-base --repo python-library + globality-build docker --repo python-library test: <<: *defaults @@ -103,14 +102,32 @@ jobs: - run: name: Run Lint command: | - docker create -v /src/microcosm-resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ eval $(aws ecr get-login --no-include-email) docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} lint - deploy: + typehinting: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + + - run: + name: Run Typehinting + command: | + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + eval $(aws ecr get-login --no-include-email) + docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} typehinting + + deploy_pypi: <<: *defaults steps: - attach_workspace: @@ -120,9 +137,10 @@ jobs: command: | echo "[distutils]" > ~/.pypirc echo "index-servers =" >> ~/.pypirc - echo " pypi" >> ~/.pypirc + echo " pypi " >> ~/.pypirc echo >> ~/.pypirc echo "[pypi]" >> ~/.pypirc + echo "repository:https://upload.pypi.org/legacy/" >> ~/.pypirc echo "username:$PYPI_USERNAME" >> ~/.pypirc echo "password:$PYPI_PASSWORD" >> ~/.pypirc echo >> ~/.pypirc @@ -160,12 +178,21 @@ workflows: # run for all branches and tags tags: only: /.*/ - - deploy: + - typehinting: + requires: + - build_base_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - deploy_pypi: requires: - test - lint + - typehinting filters: branches: ignore: /.*/ tags: only: /^[0-9]+(\.[0-9]+)*/ + From 8852de282a7523a293e1bb9206c41cd9204fd292 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Mon, 17 Dec 2018 17:06:41 -0800 Subject: [PATCH 07/18] Generating build files through globality-build From 1d68e6a6ac584780d66d2cc576fd3ac04c0bcb67 Mon Sep 17 00:00:00 2001 From: Jesse Myers Date: Wed, 19 Dec 2018 11:03:43 -0800 Subject: [PATCH 08/18] Fix typehinting --- microcosm_resourcesync/formatters/yaml_formatter.py | 4 ++-- setup.cfg | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/microcosm_resourcesync/formatters/yaml_formatter.py b/microcosm_resourcesync/formatters/yaml_formatter.py index 9c12886..88256de 100644 --- a/microcosm_resourcesync/formatters/yaml_formatter.py +++ b/microcosm_resourcesync/formatters/yaml_formatter.py @@ -4,9 +4,9 @@ """ from yaml import dump, load try: - from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader + from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader # type: ignore except ImportError: - from yaml import SafeDumper, SafeLoader + from yaml import SafeDumper, SafeLoader # type: ignore from microcosm_resourcesync.formatters.base import Formatter diff --git a/setup.cfg b/setup.cfg index 46aea63..a158556 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,9 @@ [flake8] max-line-length = 120 max-complexity = 15 + +[mypy] +ignore_missing_imports = True + +[wheel] +universal = 1 From ccf84f544b288919bcfb29eeaab31e9e01dabc05 Mon Sep 17 00:00:00 2001 From: Jesse Myers Date: Wed, 19 Dec 2018 11:04:09 -0800 Subject: [PATCH 09/18] Ignore mypy_cache --- .ackrc | 1 + .gitignore | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.ackrc b/.ackrc index 5894ec8..be53c66 100644 --- a/.ackrc +++ b/.ackrc @@ -9,6 +9,7 @@ # Python project settings --ignore-dir=.eggs/ +--ignore-dir=.mypy_cache/ --ignore-dir=.tox/ --ignore-dir=build/ --ignore-dir=cover/ diff --git a/.gitignore b/.gitignore index 8ef538f..e71834f 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,5 @@ docs/_build/ # PyBuilder target/ + +.mypy_cache From 3ec53b2c847130a5f833a869dfddeeb1235036f2 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Wed, 8 Jul 2020 14:27:59 -0700 Subject: [PATCH 10/18] Update repository metadata This is an automatic update by `terraform-github` `metadata upload` [skip ci] --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..608b207 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# This file is managed by `terraform-github` `metadata upload` +# Do not manipulate this file manually or it will be overwritten +# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners + +* @globality-corp/backend-codeowners From 0062820dee1d7aef38b7b324bfdc480e295391f3 Mon Sep 17 00:00:00 2001 From: Avi Zurel Date: Wed, 8 Jul 2020 15:04:18 -0700 Subject: [PATCH 11/18] Update repository metadata This is an automatic update by `terraform-github` `metadata upload` [skip ci] --- .github/CODEOWNERS | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 608b207..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,5 +0,0 @@ -# This file is managed by `terraform-github` `metadata upload` -# Do not manipulate this file manually or it will be overwritten -# https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners - -* @globality-corp/backend-codeowners From bae4ba3b866cc8ab8c2132a5430d05280d13d647 Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Mon, 21 Sep 2020 05:36:54 -1000 Subject: [PATCH 12/18] remove enum34 --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 9af5a26..5e65b83 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ keywords="microcosm", install_requires=[ "click>=6.7", - "enum34>=1.1.6", "PyYAML>=3.12", "requests>=2.18.4", ], From d32f11bc350af3c1f6c56e4814d44a30db80aebc Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Mon, 21 Sep 2020 05:44:59 -1000 Subject: [PATCH 13/18] update build process --- .circleci/config.yml | 112 +++++++++++++----- .dockerignore | 2 +- .globality/build.json | 7 +- Dockerfile | 112 ++++++++++++++++++ Dockerfile.template | 39 ------ .../docker-base/Dockerfile.deps | 36 ------ .../docker-base/Dockerfile.template | 60 ---------- entrypoint.sh | 27 ++--- setup.cfg | 24 +++- 9 files changed, 238 insertions(+), 181 deletions(-) create mode 100644 Dockerfile delete mode 100644 Dockerfile.template delete mode 100644 build.python-library/docker-base/Dockerfile.deps delete mode 100644 build.python-library/docker-base/Dockerfile.template diff --git a/.circleci/config.yml b/.circleci/config.yml index 31b0049..d1ec8ae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ defaults: &defaults working_directory: ~/repo docker: - - image: ${AWS_ECR_DOMAIN}/globality-build:2.0.0 + - image: ${AWS_ECR_DOMAIN}/globality-build:2020.37.0 aws_auth: aws_access_key_id: ${AWS_ACCESS_KEY_ID} aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} @@ -28,7 +28,7 @@ defaults: &defaults deploy_defaults: &deploy_defaults working_directory: ~/repo docker: - - image: ${AWS_ECR_DOMAIN}/globality-build:2.0.0 + - image: ${AWS_ECR_DOMAIN}/globality-build:2020.37.0 aws_auth: aws_access_key_id: ${AWS_ACCESS_KEY_ID} aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} @@ -39,6 +39,7 @@ deploy_defaults: &deploy_defaults PYPI_USERNAME: "InjectedDuringRuntime" PYPI_PASSWORD: "InjectedDuringRuntime" + whitelist: &whitelist paths: . @@ -56,22 +57,34 @@ jobs: root: ~/repo <<: *whitelist - build_base_docker: + build_docker: <<: *defaults steps: - attach_workspace: at: ~/repo - - setup_remote_docker + - setup_remote_docker: + docker_layer_caching: true - run: - name: Build Base Docker - # install dependencies for loading ecs task definitions + name: Login AWS ECR command: | eval $(aws ecr get-login --no-include-email) - globality-build docker-base --repo python-library - globality-build docker --repo python-library + - run: + name: Build Docker - Application Service Code + command: | + # pwd is here to prevent error when pre_docker_build returns nothing + pwd + + + docker build --tag $AWS_ECR_DOMAIN/python-library:$CIRCLE_SHA1 \ + --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM \ + --build-arg SHA1=$CIRCLE_SHA1 \ + --build-arg EXTRA_INDEX_URL=$EXTRA_INDEX_URL \ + --build-arg JFROG_AUTH=$JFROG_AUTH . + + docker push $AWS_ECR_DOMAIN/python-library:$CIRCLE_SHA1 test: <<: *defaults @@ -82,14 +95,26 @@ jobs: - setup_remote_docker - run: - name: Test code + name: Login AWS ECR command: | - docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true - docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ eval $(aws ecr get-login --no-include-email) - docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + + + - run: + name: Copy service tests to volume + command: | + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + + - run: + name: Run Test + command: | docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} test + + + lint: <<: *defaults @@ -98,14 +123,21 @@ jobs: at: ~/repo - setup_remote_docker + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) - run: - name: Run Lint + name: Copy service tests to volume command: | - docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ - eval $(aws ecr get-login --no-include-email) - docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + + - run: + name: Run Lint + command: | docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} lint @@ -117,23 +149,40 @@ jobs: at: ~/repo - setup_remote_docker + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) - run: - name: Run Typehinting + name: Copy service tests to volume command: | - docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.4 /bin/true + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ - eval $(aws ecr get-login --no-include-email) - docker pull ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} + + - run: + name: Run Typehinting + command: | docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} typehinting - deploy_pypi: + deploy_jfrog_rc: <<: *defaults steps: - attach_workspace: at: ~/repo - run: name: Deploy + command: | + echo "Not publishing package!" + + publish_library: + <<: *defaults + steps: + - attach_workspace: + at: ~/repo + - run: + name: Publish command: | echo "[distutils]" > ~/.pypirc echo "index-servers =" >> ~/.pypirc @@ -144,8 +193,11 @@ jobs: echo "username:$PYPI_USERNAME" >> ~/.pypirc echo "password:$PYPI_PASSWORD" >> ~/.pypirc echo >> ~/.pypirc + version=$(cat .bumpversion.cfg | awk '/current_version / {print $3}') python setup.py register -r pypi - python setup.py sdist upload -r pypi + python setup.py sdist + twine upload --repository pypi dist/microcosm-resourcesync-${version}.tar.gz + workflows: version: 2 @@ -157,7 +209,7 @@ workflows: # run for all branches and tags tags: only: /.*/ - - build_base_docker: + - build_docker: requires: - checkout filters: @@ -166,26 +218,31 @@ workflows: only: /.*/ - lint: requires: - - build_base_docker + - build_docker filters: # run for all branches and tags tags: only: /.*/ - test: requires: - - build_base_docker + - build_docker filters: # run for all branches and tags tags: only: /.*/ + - deploy_jfrog_rc: + requires: + - test + - lint + - typehinting - typehinting: requires: - - build_base_docker + - build_docker filters: # run for all branches and tags tags: only: /.*/ - - deploy_pypi: + - publish_library: requires: - test - lint @@ -196,3 +253,4 @@ workflows: tags: only: /^[0-9]+(\.[0-9]+)*/ + diff --git a/.dockerignore b/.dockerignore index e2582fd..a05a14d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ * !microcosm_resourcesync -microcosm_resourcesync/test* +microcosm_resourcesync/tests !dist/*-none-any.whl !entrypoint.sh !MANIFEST.in diff --git a/.globality/build.json b/.globality/build.json index b550b6c..e84105a 100644 --- a/.globality/build.json +++ b/.globality/build.json @@ -1,7 +1,10 @@ { "params": { - "name": "microcosm-resourcesync" + "name": "microcosm-resourcesync", + "pypi": { + "repository": "pypi" + } }, "type": "python-library", - "version": "2.0.36" + "version": "2020.37.0" } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..54b5aca --- /dev/null +++ b/Dockerfile @@ -0,0 +1,112 @@ +# +# Globality autogenerated Docker configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + +# ----------- deps ----------- +# Install from Debian Stretch with modern Python support +FROM python:slim-stretch as deps + +# +# Most services will use the same set of packages here, though a few will install +# custom dependencies for native requirements. +# + +ARG EXTRA_INDEX_URL +ENV EXTRA_INDEX_URL ${EXTRA_INDEX_URL} + +ENV CORE_PACKAGES locales +ENV BUILD_PACKAGES build-essential libffi-dev +ENV OTHER_PACKAGES libssl-dev + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ${CORE_PACKAGES} ${BUILD_PACKAGES} && \ + apt-get install -y --no-install-recommends ${OTHER_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + + +# ----------- base ----------- + +FROM deps as base + +# Install dependencies +# +# Since many Python distributions require a compile for their native dependencies +# we install a compiler and any required development libraries before the installation +# and then *remove* the the compiler when we are done. +# +# We can control dependency freezing by managing the contents of `requirements.txt`. +# +# We can speed up the installation a little bit by breaking out the common +# pip dependencies into their own layer, but avoid this optimization for +# now to improve clarity. +# +# We also install the web application server (which should not be one of our +# explicit dependencies). +# +# Many services will need to modify this step for Python libraries with other +# native dependencies. + + +# Work in /src +# +# We'll copy local source code here for development. +WORKDIR src + +# Set a proper locale +# +# UTF-8 everywhere. + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen "en_US.UTF-8" && \ + /usr/sbin/update-locale LANG=en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Install top-level files +# +# These are enough to install dependencies and have a stable base layer +# when source code changes. + +COPY README.md MANIFEST.in setup.cfg setup.py /src/ + +RUN pip install --no-cache-dir --upgrade --extra-index-url ${EXTRA_INDEX_URL} /src/ && \ + apt-get remove --purge -y ${BUILD_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + + +# ----------- final ----------- +FROM base + +# Setup invocation +# +# We expose the application on the standard HTTP port and use an entrypoint +# to customize the `dev` and `test` targets. + +ENV NAME microcosm_resourcesync +COPY entrypoint.sh /src/ +ENTRYPOINT ["./entrypoint.sh"] + +# Install source +# +# We should not need to reinstall dependencies here, but we do need to import +# the distribution properly. We also save build arguments to the image using +# microcosm-compatible environment variables. + + +ARG BUILD_NUM +ARG SHA1 +ENV MICROCOSM_RESOURCESYNC__BUILD_INFO_CONVENTION__BUILD_NUM ${BUILD_NUM} +ENV MICROCOSM_RESOURCESYNC__BUILD_INFO_CONVENTION__SHA1 ${SHA1} +COPY $NAME /src/$NAME/ +RUN pip install --no-cache-dir --extra-index-url $EXTRA_INDEX_URL -e . diff --git a/Dockerfile.template b/Dockerfile.template deleted file mode 100644 index 676df9d..0000000 --- a/Dockerfile.template +++ /dev/null @@ -1,39 +0,0 @@ -# -# Globality autogenerated Docker configuration -# -# This file is auto generated with globality-build. -# You should not make any changes to this file manually -# -# Any changes made to this file will be overwritten in the -# next version of the build. -# -# See: http://github.com/globality-corp/globality-build -# -# - - -FROM /python-library:base- - -# Setup invocation -# -# We expose the application on the standard HTTP port and use an entrypoint -# to customize the `dev` and `test` targets. - -ENV NAME microcosm_resourcesync -COPY README.md entrypoint.sh /src/ -ENTRYPOINT ["./entrypoint.sh"] - -# Install source -# -# We should not need to reinstall dependencies here, but we do need to import -# the distribution properly. We also save build arguments to the image using -# microcosm-compatible environment variables. - - -ARG BUILD_NUM -ARG SHA1 -ENV MICROCOSM-RESOURCESYNC__BUILD_INFO_CONVENTION__BUILD_NUM ${BUILD_NUM} -ENV MICROCOSM-RESOURCESYNC__BUILD_INFO_CONVENTION__SHA1 ${SHA1} -COPY $NAME /src/$NAME/ -RUN pip install --extra-index-url $EXTRA_INDEX_URL -e . - diff --git a/build.python-library/docker-base/Dockerfile.deps b/build.python-library/docker-base/Dockerfile.deps deleted file mode 100644 index 3e7a78f..0000000 --- a/build.python-library/docker-base/Dockerfile.deps +++ /dev/null @@ -1,36 +0,0 @@ -# -# Globality autogenerated Docker configuration -# -# This file is auto generated with globality-build. -# You should not make any changes to this file manually -# -# Any changes made to this file will be overwritten in the -# next version of the build. -# -# See: http://github.com/globality-corp/globality-build -# -# - - -# Install from Debian Stretch with modern Python support -FROM python:slim-stretch - -# -# Most services will use the same set of packages here, though a few will install -# custom dependencies for native requirements. -# - -ARG EXTRA_INDEX_URL -ENV EXTRA_INDEX_URL ${EXTRA_INDEX_URL} - -ENV CORE_PACKAGES locales -ENV BUILD_PACKAGES build-essential libffi-dev -ENV OTHER_PACKAGES libssl-dev - - -RUN apt-get update && \ - apt-get install -y --no-install-recommends ${CORE_PACKAGES} ${BUILD_PACKAGES} && \ - apt-get install -y --no-install-recommends ${OTHER_PACKAGES} && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - diff --git a/build.python-library/docker-base/Dockerfile.template b/build.python-library/docker-base/Dockerfile.template deleted file mode 100644 index 3469d92..0000000 --- a/build.python-library/docker-base/Dockerfile.template +++ /dev/null @@ -1,60 +0,0 @@ -# -# Globality autogenerated Docker configuration -# -# This file is auto generated with globality-build. -# You should not make any changes to this file manually -# -# Any changes made to this file will be overwritten in the -# next version of the build. -# -# See: http://github.com/globality-corp/globality-build -# -# - -FROM - -# Install dependencies -# -# Since many Python distributions require a compile for their native dependencies -# we install a compiler and any required development libraries before the installation -# and then *remove* the the compiler when we are done. -# -# We can control dependency freezing by managing the contents of `requirements.txt`. -# -# We can speed up the installation a little bit by breaking out the common -# pip dependencies into their own layer, but avoid this optimization for -# now to improve clarity. -# -# We also install the web application server (which should not be one of our -# explicit dependencies). -# -# Many services will need to modify this step for Python libraries with other -# native dependencies. - - -# Work in /src -# -# We'll copy local source code here for development. -WORKDIR src - -# Set a proper locale -# -# UTF-8 everywhere. - -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen "en_US.UTF-8" && \ - /usr/sbin/update-locale LANG=en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 - -# Install top-level files -# -# These are enough to install dependencies and have a stable base layer -# when source code changes. - -COPY MANIFEST.in setup.cfg setup.py /src/ - -RUN pip install --upgrade --extra-index-url ${EXTRA_INDEX_URL} /src/ && \ - apt-get remove --purge -y ${BUILD_PACKAGES} && \ - apt-get autoremove -y && \ - rm -rf /var/lib/apt/lists/* - diff --git a/entrypoint.sh b/entrypoint.sh index 2b522a8..b0b4184 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,7 +5,6 @@ # Entrypoint conventions are as follows: # # - If the container is run without a custom CMD, the service should run as it would in production. -# # - If the container is run with the "dev" CMD, the service should run in development mode. # # Normally, this means that if the user's source has been mounted as a volume, the server will @@ -20,20 +19,20 @@ if [ "$1" = "test" ]; then - # Install standard test dependencies; YMMV - pip --quiet install \ - .[test] nose mock PyHamcrest coverage - exec nosetests ${NAME} + # Install standard test dependencies; YMMV + pip --quiet install \ + .[test] nose "PyHamcrest<1.10.0" coverage + exec nosetests elif [ "$1" = "lint" ]; then - # Install standard linting dependencies; YMMV - pip --quiet install \ - .[lint] flake8 flake8-print flake8-logging-format - exec flake8 ${NAME} + # Install standard linting dependencies; YMMV + pip --quiet install \ + .[lint] flake8 flake8-print flake8-logging-format flake8-isort + exec flake8 ${NAME} elif [ "$1" = "typehinting" ]; then - # Install standard type-linting dependencies - pip --quiet install mypy - exec mypy ${NAME} --ignore-missing-imports + # Install standard type-linting dependencies + pip --quiet install mypy + mypy ${NAME} --ignore-missing-imports else - echo "Cannot execute $@" - exit 3 + echo "Cannot execute $@" + exit 3 fi diff --git a/setup.cfg b/setup.cfg index a158556..b1d732b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,29 @@ [flake8] +exclude = */migrations/*,.eggs/* max-line-length = 120 max-complexity = 15 +[isort] +combine_as_imports = True +enforce_white_space = True +force_grid_wrap = 4 +include_trailing_comma = True +known_first_party = microcosm_resourcesync +known_standard_library = dataclasses,pkg_resources +known_third_party = alembic,allennlp,boto3,click,hamcrest,joblib,matplotlib,microcosm,microcosm_sagemaker,networkx,node2vec,nose,numpy,pandas,parameterized,seaborn,six,sklearn,taxonomies,tf,torch,tqdm,unidecode +line_length = 99 +lines_after_imports = 2 +multi_line_output = 3 + [mypy] ignore_missing_imports = True -[wheel] -universal = 1 +[nosetests] +with-coverage = True +cover-package = microcosm_resourcesync +cover-html = True +cover-html-dir = coverage +cover-erase = True + +[coverage:report] +show_missing = True From 9165e9f15bef2a230a6f8585061e892638293a4d Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Mon, 21 Sep 2020 05:54:39 -1000 Subject: [PATCH 14/18] pin isort --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 5e65b83..821b9fb 100755 --- a/setup.py +++ b/setup.py @@ -20,6 +20,8 @@ "click>=6.7", "PyYAML>=3.12", "requests>=2.18.4", + # 09-21-2020: pin for enforce_white_space issue @pierce + "isort<5", ], setup_requires=[ "nose>=1.3.7", From 54bf8cb14b7e9dbc93d0eb8f29ee3bcc6f7fb67c Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Mon, 21 Sep 2020 10:42:32 -1000 Subject: [PATCH 15/18] isort --- .../endpoints/directory_endpoint.py | 7 ++++++- microcosm_resourcesync/endpoints/http_endpoint.py | 2 +- microcosm_resourcesync/formatters/yaml_formatter.py | 5 ++++- microcosm_resourcesync/main.py | 2 +- .../tests/endpoints/test_http_endpoint.py | 2 +- microcosm_resourcesync/tests/test_batching.py | 7 ++----- microcosm_resourcesync/tests/test_formatters.py | 6 +----- microcosm_resourcesync/tests/test_schemas.py | 11 ++--------- microcosm_resourcesync/tests/test_toposort.py | 5 +---- setup.cfg | 2 +- setup.py | 1 + 11 files changed, 21 insertions(+), 29 deletions(-) diff --git a/microcosm_resourcesync/endpoints/directory_endpoint.py b/microcosm_resourcesync/endpoints/directory_endpoint.py index a9f2369..72ec25c 100644 --- a/microcosm_resourcesync/endpoints/directory_endpoint.py +++ b/microcosm_resourcesync/endpoints/directory_endpoint.py @@ -3,7 +3,12 @@ """ from os import walk -from os.path import exists, isdir, join, splitext +from os.path import ( + exists, + isdir, + join, + splitext, +) from shutil import rmtree from click import ClickException diff --git a/microcosm_resourcesync/endpoints/http_endpoint.py b/microcosm_resourcesync/endpoints/http_endpoint.py index 109d5e8..b905b1c 100644 --- a/microcosm_resourcesync/endpoints/http_endpoint.py +++ b/microcosm_resourcesync/endpoints/http_endpoint.py @@ -3,8 +3,8 @@ """ from os.path import commonprefix -from urllib.parse import urlparse, urlunparse from sys import stderr +from urllib.parse import urlparse, urlunparse from click import echo, progressbar from requests import Session diff --git a/microcosm_resourcesync/formatters/yaml_formatter.py b/microcosm_resourcesync/formatters/yaml_formatter.py index 88256de..ff4a4b8 100644 --- a/microcosm_resourcesync/formatters/yaml_formatter.py +++ b/microcosm_resourcesync/formatters/yaml_formatter.py @@ -3,12 +3,15 @@ """ from yaml import dump, load + +from microcosm_resourcesync.formatters.base import Formatter + + try: from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader # type: ignore except ImportError: from yaml import SafeDumper, SafeLoader # type: ignore -from microcosm_resourcesync.formatters.base import Formatter class YAMLFormatter(Formatter): diff --git a/microcosm_resourcesync/main.py b/microcosm_resourcesync/main.py index 809b43d..3ac23c3 100644 --- a/microcosm_resourcesync/main.py +++ b/microcosm_resourcesync/main.py @@ -3,8 +3,8 @@ """ from click import ( - argument, BadParameter, + argument, command, echo, option, diff --git a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py index fc75fe4..4661901 100644 --- a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py +++ b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py @@ -3,7 +3,7 @@ """ from json import dumps -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch from hamcrest import ( assert_that, diff --git a/microcosm_resourcesync/tests/test_batching.py b/microcosm_resourcesync/tests/test_batching.py index 95c7fd3..8249afa 100644 --- a/microcosm_resourcesync/tests/test_batching.py +++ b/microcosm_resourcesync/tests/test_batching.py @@ -2,13 +2,10 @@ Test batching. """ -from hamcrest import ( - assert_that, - contains, -) +from hamcrest import assert_that, contains -from microcosm_resourcesync.schemas import SimpleSchema from microcosm_resourcesync.batching import batched +from microcosm_resourcesync.schemas import SimpleSchema resources = [ diff --git a/microcosm_resourcesync/tests/test_formatters.py b/microcosm_resourcesync/tests/test_formatters.py index 1f50927..b05b875 100644 --- a/microcosm_resourcesync/tests/test_formatters.py +++ b/microcosm_resourcesync/tests/test_formatters.py @@ -2,11 +2,7 @@ Formatter tests. """ -from hamcrest import ( - assert_that, - equal_to, - is_, -) +from hamcrest import assert_that, equal_to, is_ from microcosm_resourcesync.formatters import Formatters diff --git a/microcosm_resourcesync/tests/test_schemas.py b/microcosm_resourcesync/tests/test_schemas.py index afe250d..0f2b758 100644 --- a/microcosm_resourcesync/tests/test_schemas.py +++ b/microcosm_resourcesync/tests/test_schemas.py @@ -2,16 +2,9 @@ Schema tests. """ -from hamcrest import ( - assert_that, - equal_to, - is_, -) +from hamcrest import assert_that, equal_to, is_ -from microcosm_resourcesync.schemas import ( - HALSchema, - SimpleSchema, -) +from microcosm_resourcesync.schemas import HALSchema, SimpleSchema ID = "c7f12ba5885f4b47bfafaa583cd5a097" diff --git a/microcosm_resourcesync/tests/test_toposort.py b/microcosm_resourcesync/tests/test_toposort.py index f1332da..ca37206 100644 --- a/microcosm_resourcesync/tests/test_toposort.py +++ b/microcosm_resourcesync/tests/test_toposort.py @@ -4,10 +4,7 @@ """ from random import shuffle -from hamcrest import ( - assert_that, - contains, -) +from hamcrest import assert_that, contains from microcosm_resourcesync.schemas import SimpleSchema from microcosm_resourcesync.toposort import toposorted diff --git a/setup.cfg b/setup.cfg index b1d732b..3e784e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ force_grid_wrap = 4 include_trailing_comma = True known_first_party = microcosm_resourcesync known_standard_library = dataclasses,pkg_resources -known_third_party = alembic,allennlp,boto3,click,hamcrest,joblib,matplotlib,microcosm,microcosm_sagemaker,networkx,node2vec,nose,numpy,pandas,parameterized,seaborn,six,sklearn,taxonomies,tf,torch,tqdm,unidecode +known_third_party = alembic,allennlp,boto3,click,hamcrest,joblib,matplotlib,microcosm,microcosm_sagemaker,networkx,node2vec,nose,numpy,pandas,parameterized,seaborn,six,sklearn,taxonomies,tf,torch,tqdm,unidecode,requests line_length = 99 lines_after_imports = 2 multi_line_output = 3 diff --git a/setup.py b/setup.py index 821b9fb..47fdf51 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from setuptools import find_packages, setup + project = "microcosm-resourcesync" version = "1.0.0" From 9006fc0d880db4c06dd52a0bf6221d425f55d467 Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Mon, 21 Sep 2020 17:12:35 -1000 Subject: [PATCH 16/18] fix lint --- microcosm_resourcesync/formatters/yaml_formatter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/microcosm_resourcesync/formatters/yaml_formatter.py b/microcosm_resourcesync/formatters/yaml_formatter.py index ff4a4b8..38d3146 100644 --- a/microcosm_resourcesync/formatters/yaml_formatter.py +++ b/microcosm_resourcesync/formatters/yaml_formatter.py @@ -13,7 +13,6 @@ from yaml import SafeDumper, SafeLoader # type: ignore - class YAMLFormatter(Formatter): def load(self, data): From 72733a839f3ec7bf498d161cdd89489af83f51a3 Mon Sep 17 00:00:00 2001 From: Pierce Freeman Date: Tue, 22 Sep 2020 09:14:52 -1000 Subject: [PATCH 17/18] move isort to entrypoint --- entrypoint.sh | 2 +- setup.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index b0b4184..659f2d3 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -26,7 +26,7 @@ if [ "$1" = "test" ]; then elif [ "$1" = "lint" ]; then # Install standard linting dependencies; YMMV pip --quiet install \ - .[lint] flake8 flake8-print flake8-logging-format flake8-isort + .[lint] flake8 flake8-print flake8-logging-format flake8-isort "isort<5.0.0" exec flake8 ${NAME} elif [ "$1" = "typehinting" ]; then # Install standard type-linting dependencies diff --git a/setup.py b/setup.py index 47fdf51..5ff5d40 100755 --- a/setup.py +++ b/setup.py @@ -21,8 +21,6 @@ "click>=6.7", "PyYAML>=3.12", "requests>=2.18.4", - # 09-21-2020: pin for enforce_white_space issue @pierce - "isort<5", ], setup_requires=[ "nose>=1.3.7", From 0a205164189cb116cc5db25c82391e15868f73cf Mon Sep 17 00:00:00 2001 From: Adam Ever-Hadani Date: Tue, 22 Sep 2020 12:25:04 -0700 Subject: [PATCH 18/18] bump version --- .bumpversion.cfg | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ebab3c1..afc26ec 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,9 +1,8 @@ [bumpversion] -current_version = 1.0.0 +current_version = 1.0.1 commit = False tag = False [bumpversion:file:setup.py] search = version = "{current_version}" replace = version = "{new_version}" - diff --git a/setup.py b/setup.py index 5ff5d40..07a7a70 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ project = "microcosm-resourcesync" -version = "1.0.0" +version = "1.0.1" setup( name=project,