diff --git a/.ackrc b/.ackrc index 5894ec8..be53c66 100644 --- a/.ackrc +++ b/.ackrc @@ -9,6 +9,7 @@ # Python project settings --ignore-dir=.eggs/ +--ignore-dir=.mypy_cache/ --ignore-dir=.tox/ --ignore-dir=build/ --ignore-dir=cover/ diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ebab3c1..afc26ec 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,9 +1,8 @@ [bumpversion] -current_version = 1.0.0 +current_version = 1.0.1 commit = False tag = False [bumpversion:file:setup.py] search = version = "{current_version}" replace = version = "{new_version}" - diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..d1ec8ae --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,256 @@ +# +# Globality autogenerated CircleCI configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + +defaults: &defaults + working_directory: ~/repo + docker: + - image: ${AWS_ECR_DOMAIN}/globality-build:2020.37.0 + aws_auth: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + environment: + EXTRA_INDEX_URL: "InjectedDuringRuntime" + AWS_ECR_DOMAIN: "InjectedDuringRuntime" + JFROG_AUTH: "InjectedDuringRuntime" + PYPI_USERNAME: "InjectedDuringRuntime" + PYPI_PASSWORD: "InjectedDuringRuntime" + +deploy_defaults: &deploy_defaults + working_directory: ~/repo + docker: + - image: ${AWS_ECR_DOMAIN}/globality-build:2020.37.0 + aws_auth: + aws_access_key_id: ${AWS_ACCESS_KEY_ID} + aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY} + environment: + EXTRA_INDEX_URL: "InjectedDuringRuntime" + AWS_ECR_DOMAIN: "InjectedDuringRuntime" + JFROG_AUTH: "InjectedDuringRuntime" + PYPI_USERNAME: "InjectedDuringRuntime" + PYPI_PASSWORD: "InjectedDuringRuntime" + + +whitelist: &whitelist + paths: + . + +version: 2 + +jobs: + checkout: + <<: *defaults + + steps: + - checkout + + - persist_to_workspace: + root: ~/repo + <<: *whitelist + + build_docker: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker: + docker_layer_caching: true + + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) + - run: + name: Build Docker - Application Service Code + command: | + # pwd is here to prevent error when pre_docker_build returns nothing + pwd + + + docker build --tag $AWS_ECR_DOMAIN/python-library:$CIRCLE_SHA1 \ + --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM \ + --build-arg SHA1=$CIRCLE_SHA1 \ + --build-arg EXTRA_INDEX_URL=$EXTRA_INDEX_URL \ + --build-arg JFROG_AUTH=$JFROG_AUTH . + + docker push $AWS_ECR_DOMAIN/python-library:$CIRCLE_SHA1 + + test: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) + + + - run: + name: Copy service tests to volume + command: | + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + + - run: + name: Run Test + command: | + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} test + + + + + lint: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) + + - run: + name: Copy service tests to volume + command: | + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + + - run: + name: Run Lint + command: | + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} lint + + + typehinting: + <<: *defaults + + steps: + - attach_workspace: + at: ~/repo + + - setup_remote_docker + - run: + name: Login AWS ECR + command: | + eval $(aws ecr get-login --no-include-email) + + - run: + name: Copy service tests to volume + command: | + + docker create -v /src/microcosm_resourcesync/tests/ --name service_tests alpine:3.11 /bin/true + docker cp $(pwd)/microcosm_resourcesync/tests service_tests:/src/microcosm_resourcesync/ + + - run: + name: Run Typehinting + command: | + docker run -it --volumes-from service_tests ${AWS_ECR_DOMAIN}/python-library:${CIRCLE_SHA1} typehinting + + deploy_jfrog_rc: + <<: *defaults + steps: + - attach_workspace: + at: ~/repo + - run: + name: Deploy + command: | + echo "Not publishing package!" + + publish_library: + <<: *defaults + steps: + - attach_workspace: + at: ~/repo + - run: + name: Publish + command: | + echo "[distutils]" > ~/.pypirc + echo "index-servers =" >> ~/.pypirc + echo " pypi " >> ~/.pypirc + echo >> ~/.pypirc + echo "[pypi]" >> ~/.pypirc + echo "repository:https://upload.pypi.org/legacy/" >> ~/.pypirc + echo "username:$PYPI_USERNAME" >> ~/.pypirc + echo "password:$PYPI_PASSWORD" >> ~/.pypirc + echo >> ~/.pypirc + version=$(cat .bumpversion.cfg | awk '/current_version / {print $3}') + python setup.py register -r pypi + python setup.py sdist + twine upload --repository pypi dist/microcosm-resourcesync-${version}.tar.gz + + +workflows: + version: 2 + + build-and-release: + jobs: + - checkout: + filters: + # run for all branches and tags + tags: + only: /.*/ + - build_docker: + requires: + - checkout + filters: + # run for all branches and tags + tags: + only: /.*/ + - lint: + requires: + - build_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - test: + requires: + - build_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - deploy_jfrog_rc: + requires: + - test + - lint + - typehinting + - typehinting: + requires: + - build_docker + filters: + # run for all branches and tags + tags: + only: /.*/ + - publish_library: + requires: + - test + - lint + - typehinting + filters: + branches: + ignore: /.*/ + tags: + only: /^[0-9]+(\.[0-9]+)*/ + + diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..a05a14d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +* +!microcosm_resourcesync +microcosm_resourcesync/tests +!dist/*-none-any.whl +!entrypoint.sh +!MANIFEST.in +!README.md +!requirements.txt +!setup.py +!setup.cfg +**/*.pyc +**/*~ + diff --git a/.gitignore b/.gitignore index 8ef538f..e71834f 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,5 @@ docs/_build/ # PyBuilder target/ + +.mypy_cache diff --git a/.globality/build.json b/.globality/build.json new file mode 100644 index 0000000..e84105a --- /dev/null +++ b/.globality/build.json @@ -0,0 +1,10 @@ +{ + "params": { + "name": "microcosm-resourcesync", + "pypi": { + "repository": "pypi" + } + }, + "type": "python-library", + "version": "2020.37.0" +} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..54b5aca --- /dev/null +++ b/Dockerfile @@ -0,0 +1,112 @@ +# +# Globality autogenerated Docker configuration +# +# This file is auto generated with globality-build. +# You should not make any changes to this file manually +# +# Any changes made to this file will be overwritten in the +# next version of the build. +# +# See: http://github.com/globality-corp/globality-build +# +# + +# ----------- deps ----------- +# Install from Debian Stretch with modern Python support +FROM python:slim-stretch as deps + +# +# Most services will use the same set of packages here, though a few will install +# custom dependencies for native requirements. +# + +ARG EXTRA_INDEX_URL +ENV EXTRA_INDEX_URL ${EXTRA_INDEX_URL} + +ENV CORE_PACKAGES locales +ENV BUILD_PACKAGES build-essential libffi-dev +ENV OTHER_PACKAGES libssl-dev + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends ${CORE_PACKAGES} ${BUILD_PACKAGES} && \ + apt-get install -y --no-install-recommends ${OTHER_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + + +# ----------- base ----------- + +FROM deps as base + +# Install dependencies +# +# Since many Python distributions require a compile for their native dependencies +# we install a compiler and any required development libraries before the installation +# and then *remove* the the compiler when we are done. +# +# We can control dependency freezing by managing the contents of `requirements.txt`. +# +# We can speed up the installation a little bit by breaking out the common +# pip dependencies into their own layer, but avoid this optimization for +# now to improve clarity. +# +# We also install the web application server (which should not be one of our +# explicit dependencies). +# +# Many services will need to modify this step for Python libraries with other +# native dependencies. + + +# Work in /src +# +# We'll copy local source code here for development. +WORKDIR src + +# Set a proper locale +# +# UTF-8 everywhere. + +RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ + locale-gen "en_US.UTF-8" && \ + /usr/sbin/update-locale LANG=en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +# Install top-level files +# +# These are enough to install dependencies and have a stable base layer +# when source code changes. + +COPY README.md MANIFEST.in setup.cfg setup.py /src/ + +RUN pip install --no-cache-dir --upgrade --extra-index-url ${EXTRA_INDEX_URL} /src/ && \ + apt-get remove --purge -y ${BUILD_PACKAGES} && \ + apt-get autoremove -y && \ + rm -rf /var/lib/apt/lists/* + + +# ----------- final ----------- +FROM base + +# Setup invocation +# +# We expose the application on the standard HTTP port and use an entrypoint +# to customize the `dev` and `test` targets. + +ENV NAME microcosm_resourcesync +COPY entrypoint.sh /src/ +ENTRYPOINT ["./entrypoint.sh"] + +# Install source +# +# We should not need to reinstall dependencies here, but we do need to import +# the distribution properly. We also save build arguments to the image using +# microcosm-compatible environment variables. + + +ARG BUILD_NUM +ARG SHA1 +ENV MICROCOSM_RESOURCESYNC__BUILD_INFO_CONVENTION__BUILD_NUM ${BUILD_NUM} +ENV MICROCOSM_RESOURCESYNC__BUILD_INFO_CONVENTION__SHA1 ${SHA1} +COPY $NAME /src/$NAME/ +RUN pip install --no-cache-dir --extra-index-url $EXTRA_INDEX_URL -e . diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e69de29 diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 1bff0ee..0000000 --- a/circle.yml +++ /dev/null @@ -1,35 +0,0 @@ -# CircleCI Configuration file - -machine: - python: - version: 3.6.2 - -general: - artifacts: - - "dist" - - "cover" - -dependencies: - override: - - pip install tox tox-pyenv - - pyenv local 3.6.2 - -test: - override: - - tox - -deployment: - pypi: - tag: /[0-9]+(\.[0-9]+)*/ - owner: globality-corp - commands: - - echo "[distutils]" > ~/.pypirc - - echo "index-servers =" >> ~/.pypirc - - echo " pypi" >> ~/.pypirc - - echo >> ~/.pypirc - - echo "[pypi]" >> ~/.pypirc - - echo "username:$PYPI_USERNAME" >> ~/.pypirc - - echo "password:$PYPI_PASSWORD" >> ~/.pypirc - - echo >> ~/.pypirc - - python setup.py register -r pypi - - python setup.py sdist upload -r pypi diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..659f2d3 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/bash -e + +# Container entrypoint to simplify running the production and dev servers. + +# Entrypoint conventions are as follows: +# +# - If the container is run without a custom CMD, the service should run as it would in production. +# - If the container is run with the "dev" CMD, the service should run in development mode. +# +# Normally, this means that if the user's source has been mounted as a volume, the server will +# restart on code changes and will inject extra debugging/diagnostics. +# +# - If the CMD is "test" or "lint", the service should run its unit tests or linting. +# +# There is no requirement that unit tests work unless user source has been mounted as a volume; +# test code should not normally be shipped with production images. +# +# - Otherwise, the CMD should be run verbatim. + + +if [ "$1" = "test" ]; then + # Install standard test dependencies; YMMV + pip --quiet install \ + .[test] nose "PyHamcrest<1.10.0" coverage + exec nosetests +elif [ "$1" = "lint" ]; then + # Install standard linting dependencies; YMMV + pip --quiet install \ + .[lint] flake8 flake8-print flake8-logging-format flake8-isort "isort<5.0.0" + exec flake8 ${NAME} +elif [ "$1" = "typehinting" ]; then + # Install standard type-linting dependencies + pip --quiet install mypy + mypy ${NAME} --ignore-missing-imports +else + echo "Cannot execute $@" + exit 3 +fi diff --git a/microcosm_resourcesync/endpoints/directory_endpoint.py b/microcosm_resourcesync/endpoints/directory_endpoint.py index a9f2369..72ec25c 100644 --- a/microcosm_resourcesync/endpoints/directory_endpoint.py +++ b/microcosm_resourcesync/endpoints/directory_endpoint.py @@ -3,7 +3,12 @@ """ from os import walk -from os.path import exists, isdir, join, splitext +from os.path import ( + exists, + isdir, + join, + splitext, +) from shutil import rmtree from click import ClickException diff --git a/microcosm_resourcesync/endpoints/http_endpoint.py b/microcosm_resourcesync/endpoints/http_endpoint.py index 109d5e8..b905b1c 100644 --- a/microcosm_resourcesync/endpoints/http_endpoint.py +++ b/microcosm_resourcesync/endpoints/http_endpoint.py @@ -3,8 +3,8 @@ """ from os.path import commonprefix -from urllib.parse import urlparse, urlunparse from sys import stderr +from urllib.parse import urlparse, urlunparse from click import echo, progressbar from requests import Session diff --git a/microcosm_resourcesync/formatters/yaml_formatter.py b/microcosm_resourcesync/formatters/yaml_formatter.py index 9c12886..38d3146 100644 --- a/microcosm_resourcesync/formatters/yaml_formatter.py +++ b/microcosm_resourcesync/formatters/yaml_formatter.py @@ -3,14 +3,16 @@ """ from yaml import dump, load -try: - from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader -except ImportError: - from yaml import SafeDumper, SafeLoader from microcosm_resourcesync.formatters.base import Formatter +try: + from yaml import CSafeDumper as SafeDumper, CSafeLoader as SafeLoader # type: ignore +except ImportError: + from yaml import SafeDumper, SafeLoader # type: ignore + + class YAMLFormatter(Formatter): def load(self, data): diff --git a/microcosm_resourcesync/main.py b/microcosm_resourcesync/main.py index 809b43d..3ac23c3 100644 --- a/microcosm_resourcesync/main.py +++ b/microcosm_resourcesync/main.py @@ -3,8 +3,8 @@ """ from click import ( - argument, BadParameter, + argument, command, echo, option, diff --git a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py index 4f69c4a..4661901 100644 --- a/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py +++ b/microcosm_resourcesync/tests/endpoints/test_http_endpoint.py @@ -3,6 +3,7 @@ """ from json import dumps +from unittest.mock import Mock, patch from hamcrest import ( assert_that, @@ -10,7 +11,6 @@ has_length, is_, ) -from mock import patch, Mock from microcosm_resourcesync.endpoints import HTTPEndpoint from microcosm_resourcesync.following import FollowMode @@ -143,9 +143,6 @@ def test_read_follow_page(self): self=dict( href="http://example.com/api/foo?offset=2&limit2", ), - prev=dict( - href="http://example.com/api/foo?offset=0&limit=2", - ), ), items=[ hal_resource("http://example.com/api/foo/3"), diff --git a/microcosm_resourcesync/tests/test_batching.py b/microcosm_resourcesync/tests/test_batching.py index 95c7fd3..8249afa 100644 --- a/microcosm_resourcesync/tests/test_batching.py +++ b/microcosm_resourcesync/tests/test_batching.py @@ -2,13 +2,10 @@ Test batching. """ -from hamcrest import ( - assert_that, - contains, -) +from hamcrest import assert_that, contains -from microcosm_resourcesync.schemas import SimpleSchema from microcosm_resourcesync.batching import batched +from microcosm_resourcesync.schemas import SimpleSchema resources = [ diff --git a/microcosm_resourcesync/tests/test_formatters.py b/microcosm_resourcesync/tests/test_formatters.py index 1f50927..b05b875 100644 --- a/microcosm_resourcesync/tests/test_formatters.py +++ b/microcosm_resourcesync/tests/test_formatters.py @@ -2,11 +2,7 @@ Formatter tests. """ -from hamcrest import ( - assert_that, - equal_to, - is_, -) +from hamcrest import assert_that, equal_to, is_ from microcosm_resourcesync.formatters import Formatters diff --git a/microcosm_resourcesync/tests/test_schemas.py b/microcosm_resourcesync/tests/test_schemas.py index afe250d..0f2b758 100644 --- a/microcosm_resourcesync/tests/test_schemas.py +++ b/microcosm_resourcesync/tests/test_schemas.py @@ -2,16 +2,9 @@ Schema tests. """ -from hamcrest import ( - assert_that, - equal_to, - is_, -) +from hamcrest import assert_that, equal_to, is_ -from microcosm_resourcesync.schemas import ( - HALSchema, - SimpleSchema, -) +from microcosm_resourcesync.schemas import HALSchema, SimpleSchema ID = "c7f12ba5885f4b47bfafaa583cd5a097" diff --git a/microcosm_resourcesync/tests/test_toposort.py b/microcosm_resourcesync/tests/test_toposort.py index f1332da..ca37206 100644 --- a/microcosm_resourcesync/tests/test_toposort.py +++ b/microcosm_resourcesync/tests/test_toposort.py @@ -4,10 +4,7 @@ """ from random import shuffle -from hamcrest import ( - assert_that, - contains, -) +from hamcrest import assert_that, contains from microcosm_resourcesync.schemas import SimpleSchema from microcosm_resourcesync.toposort import toposorted diff --git a/setup.cfg b/setup.cfg index 46aea63..3e784e7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,29 @@ [flake8] +exclude = */migrations/*,.eggs/* max-line-length = 120 max-complexity = 15 + +[isort] +combine_as_imports = True +enforce_white_space = True +force_grid_wrap = 4 +include_trailing_comma = True +known_first_party = microcosm_resourcesync +known_standard_library = dataclasses,pkg_resources +known_third_party = alembic,allennlp,boto3,click,hamcrest,joblib,matplotlib,microcosm,microcosm_sagemaker,networkx,node2vec,nose,numpy,pandas,parameterized,seaborn,six,sklearn,taxonomies,tf,torch,tqdm,unidecode,requests +line_length = 99 +lines_after_imports = 2 +multi_line_output = 3 + +[mypy] +ignore_missing_imports = True + +[nosetests] +with-coverage = True +cover-package = microcosm_resourcesync +cover-html = True +cover-html-dir = coverage +cover-erase = True + +[coverage:report] +show_missing = True diff --git a/setup.py b/setup.py index f92935b..07a7a70 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,9 @@ #!/usr/bin/env python from setuptools import find_packages, setup + project = "microcosm-resourcesync" -version = "1.0.0" +version = "1.0.1" setup( name=project, @@ -14,10 +15,10 @@ packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), include_package_data=True, zip_safe=False, + python_requires=">=3.6", keywords="microcosm", install_requires=[ "click>=6.7", - "enum34>=1.1.6", "PyYAML>=3.12", "requests>=2.18.4", ], @@ -33,7 +34,6 @@ }, tests_require=[ "coverage>=4.3.4", - "mock>=2.0.0", "PyHamcrest>=1.9.0", ], )