config.yml

# Python CircleCI 2.0 configuration file
#
# Check https://circleci.com/docs/2.0/language-python/ for more details
#
version: 2.1

orbs:
  gh-release: vertexproject/github-release@0.0.4

commands:
  do_report_coverage:
    description: "Codecov report upload"
    steps:
      - run:
          name: Upload Coverage Results
          command: |
            # Activate the venv so that codecov bash uploader finds coverage
            . venv/bin/activate
            # Use the codecov call from the codecov orb..
            curl -s https://codecov.io/bash | bash -s -- \
              -t "${CODECOV_TOKEN}" \
              -n "${CODECOV_PREFIX}${PYVERS}node${CIRCLE_NODE_INDEX}" \
              -F "${CODECOV_FLAG}" \
              -Z || echo 'Codecov upload failed'

  do_venv_setup:
    description: "Setup venv for testing"
    steps:
      - run:
          name: setup venv
          command: |
            python3 -m venv --copies venv
            . venv/bin/activate
            python3 -m pip install -U wheel pip pytest pytest-cov pycodestyle

      - run:
          name: install synapse
          command: |
            . venv/bin/activate
            python3 -m pip install -U --upgrade-strategy=eager -e .

  do_test_execution:
    description: "Execute unit tests via pytest"
    steps:
      - run:
          name: run tests
          command: |
            . venv/bin/activate
            mkdir test-reports
            circleci tests glob synapse/tests/test_*.py | circleci tests split --split-by=timings | xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml ${COVERAGE_ARGS}

  do_replay_test_execution:
    description: "Execute unit tests via pytest"
    steps:
      - run:
          name: run tests
          command: |
            . venv/bin/activate
            mkdir test-reports
            circleci tests glob synapse/tests/test_cortex.py synapse/tests/test_lib_storm.py synapse/tests/test_lib_stormtypes.py synapse/tests/test_lib_stormlib*.py synapse/tests/test_lib_view.py synapse/tests/test_lib_hiveauth.py synapse/tests/test_lib_cell.py synapse/tests/test_lib_layer.py | circleci tests split --split-by=timings | xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml ${COVERAGE_ARGS}

  test_steps_doc:
    description: "Documentation test steps"
    steps:
      - checkout

      - run:
          name: install deps
          command: |
            sudo apt-get -y install pandoc

      - restore_cache:
          keys:
            - v2-docvenv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: setup venv
          command: |
            python3 -m venv --copies venv
            . venv/bin/activate
            python3 -m pip install -U wheel pip

      - run:
          name: install synapse requirements
          command: |
            . venv/bin/activate
            python3 setup.py egg_info
            grep -v -E "^\[" synapse.egg-info/requires.txt > _requirements.txt
            python3 -m pip install -U --upgrade-strategy=eager -r _requirements.txt

      - save_cache:
          paths:
            - ./venv
          key: v2-docvenv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: executing docs jupyter notebooks
          command: |
            . venv/bin/activate
            ./scripts/doctests.py

  test_steps_python:
    description: "Python test steps"
    steps:
      - checkout

      - run:
          name: checkout regression repo
          command: |
            git clone https://github.com/vertexproject/synapse-regression ~/git/synapse-regression

      - run:
          name: install deps
          command: |
            sudo apt-get -y install cron

      - restore_cache:
          keys:
            - v2-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - do_venv_setup

      - save_cache:
          paths:
            - ./venv
          key: v2-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: syntax
          command: |
            . venv/bin/activate
            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle synapse; fi;
            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle scripts; fi;


      - do_test_execution
      - do_report_coverage

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports

  test_steps_python_replay:
    description: "Python test steps"
    steps:
      - checkout

      - run:
          name: checkout regression repo
          command: |
            git clone https://github.com/vertexproject/synapse-regression ~/git/synapse-regression

      - run:
          name: install deps
          command: |
            sudo apt-get -y install cron

      - restore_cache:
          keys:
            - v2-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - do_venv_setup

      - save_cache:
          paths:
            - ./venv
          key: v2-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: syntax
          command: |
            . venv/bin/activate
            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle synapse; fi;
            if [ -n "${RUN_SYNTAX}" ]; then pycodestyle scripts; fi;

      - do_replay_test_execution
      - do_report_coverage

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports

  test_steps_osx:
    description: "OSX Test Steps"
    steps:
      - checkout

      - restore_cache:
          keys:
            - v5-osx-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - do_venv_setup

      - save_cache:
          paths:
            - ./venv
          key: v5-osx-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: run tests
          command: |
            ulimit -Sn 4096
            . venv/bin/activate
            mkdir test-reports
            circleci tests glob synapse/tests/test_*.py | circleci tests split --split-by=timings | xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml ${COVERAGE_ARGS}

      - do_report_coverage

      - store_test_results:
          path: test-reports

      - store_artifacts:
          path: test-reports

  deploy_pypi_prologue:
    description: "Common Pypi prologue"
    steps:
      - checkout

      - restore_cache:
          keys:
            - v1-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: install python dependencies
          command: |
            python3 -m venv venv
            . venv/bin/activate
            python3 -m pip install -U wheel pip twine
            python3 -m pip install -U -r requirements_dev.txt

      - save_cache:
          paths:
            - ./venv
          key: v1-venv-{{ .Environment.CIRCLE_STAGE }}-{{ .Branch }}-{{ checksum "setup.py" }}

      - run:
          name: init .pypirc
          command: |
            echo -e "[pypi]" >> ~/.pypirc
            echo -e "username = $PYPI_USERNAME" >> ~/.pypirc
            echo -e "password = $PYPI_PASSWORD" >> ~/.pypirc

  deploy_pypi_tag_verify:
    description: "Verify the git tag"
    steps:
      - run:
          name: verify git tag vs. version
          command: |
            env | grep -i circle
            . venv/bin/activate
            python setup.py verify

  deploy_pypi_sdist_build:
    description: "Build sdist packages for pypi"
    steps:
      - run:
          name: create packages
          command: |
            . venv/bin/activate
            python setup.py setcommit
            python setup.py sdist

      - run:
          name: smoke packages
          command: |
            mkdir -p /tmp/sdisttest
            python3 -m venv /tmp/sdisttest/venv
            cp dist/*.tar.gz /tmp/sdisttest
            cd /tmp/sdisttest
            . ./venv/bin/activate
            python3 -m pip install -U wheel pip twine
            python3 -m twine check *.tar.gz
            python3 -m pip install *.tar.gz
            python3 -c "$PYPI_SMOKE_CODE"
            deactivate

  deploy_pypi_wheel_build:
    description: "Build wheel packages for pypi"
    steps:
      - run:
          name: create packages
          command: |
            . venv/bin/activate
            python setup.py setcommit
            python setup.py bdist_wheel --python-tag=$PYTHON_TAG

      - run:
          name: smoke packages
          command: |
            mkdir -p /tmp/wheeltest
            python3 -m venv /tmp/wheeltest/venv
            cp dist/*.whl /tmp/wheeltest
            cd /tmp/wheeltest
            . ./venv/bin/activate
            python3 -m pip install -U wheel pip twine
            python3 -m twine check *.whl
            python3 -m pip install *.whl
            python3 -c "$PYPI_SMOKE_CODE"
            deactivate

  deploy_pypi_upload:
    description: "Upload packages to pypi"
    steps:
      - run:
          name: upload to pypi
          command: |
            . venv/bin/activate
            twine upload dist/*

  check_docker_smoketests:
    description: "Check docker smoke test results"
    steps:
      - run:
          name: validate docker smoketest
          command: |
            sleep 60
            docker ps
            docker container ls -a
            docker logs test01
            docker logs test02
            docker logs test03
            docker logs test04
            dstatus01=`docker inspect test01 --format '{{.State.Health.Status}}'`
            dstatus02=`docker inspect test02 --format '{{.State.Health.Status}}'`
            dstatus03=`docker inspect test03 --format '{{.State.Health.Status}}'`
            dstatus04=`docker inspect test04 --format '{{.State.Health.Status}}'`
            docker stop test01
            docker stop test02
            docker stop test03
            docker stop test04
            docker rm test01
            docker rm test02
            docker rm test03
            docker rm test04
            if [ $dstatus01 != "healthy" ]; then exit 1; fi
            if [ $dstatus02 != "healthy" ]; then exit 1; fi
            if [ $dstatus03 != "healthy" ]; then exit 1; fi
            if [ $dstatus04 != "healthy" ]; then exit 1; fi
            exit 0

  do_docker_login:
    description: "Login to docker"
    steps:
      - run:
          name: docker login
          command: |
            docker login --username "${DOCKER_USERNAME}" --password "${DOCKER_PASSWORD}"

  deploy_docker_tags:
    description: "Deploy current tag images to docker"
    steps:
      - checkout
      - setup_remote_docker
      - do_docker_login
      - run:
          name: install python
          command: |
            apk add python3 py3-pip git
      - run:
          name: setcommit
          command: |
            python3 setup.py setcommit
      - run:
          name: build images
          command: |
            docker build -t vertexproject/synapse:${CIRCLE_TAG} -f docker/images/synapse/Dockerfile .
            docker build -t vertexproject/synapse:${CIRCLE_TAG}-py37 -f docker/images/synapse/Dockerfile-py37 .
            docker build -t vertexproject/synapse-aha:${CIRCLE_TAG} -f docker/images/aha/Dockerfile .
            docker build -t vertexproject/synapse-axon:${CIRCLE_TAG} -f docker/images/axon/Dockerfile .
            docker build -t vertexproject/synapse-cortex:${CIRCLE_TAG} -f docker/images/cortex/Dockerfile .
            docker build -t vertexproject/synapse-cryotank:${CIRCLE_TAG} -f docker/images/cryotank/Dockerfile .
      - run:
          name: smoke test image
          command: |
            docker run --rm -it --entrypoint python vertexproject/synapse:${CIRCLE_TAG} -m synapse.servers.cortex --help
            dstatus00=$?
            if [ $dstatus00 != "0" ]; then exit 1; fi
            docker run --rm -it --entrypoint python vertexproject/synapse:${CIRCLE_TAG}-py37 -m synapse.servers.cortex --help
            dstatus01=$?
            if [ $dstatus01 != "0" ]; then exit 1; fi
            docker run -d --name test01 vertexproject/synapse-aha:${CIRCLE_TAG}
            docker run -d --name test02 vertexproject/synapse-axon:${CIRCLE_TAG}
            docker run -d --name test03 vertexproject/synapse-cortex:${CIRCLE_TAG}
            docker run -d --name test04 vertexproject/synapse-cryotank:${CIRCLE_TAG}
      - check_docker_smoketests
      - run:
          name: tag and push container
          command: |
            docker push vertexproject/synapse:${CIRCLE_TAG}
            docker push vertexproject/synapse:${CIRCLE_TAG}-py37
            docker push vertexproject/synapse-aha:${CIRCLE_TAG}
            docker push vertexproject/synapse-axon:${CIRCLE_TAG}
            docker push vertexproject/synapse-cortex:${CIRCLE_TAG}
            docker push vertexproject/synapse-cryotank:${CIRCLE_TAG}

  deploy_docker_2xx:
    description: "Build current release tag to docker as v2.x.x"
    steps:
      - checkout
      - setup_remote_docker
      - do_docker_login
      - run:
          name: install python
          command: |
            apk add python3 py3-pip git
      - run:
          name: setcommit
          command: |
            python3 setup.py setcommit
      - run:
          name: build image
          command: |
            docker build -t vertexproject/synapse:v2.x.x -f docker/images/synapse/Dockerfile .
            docker build -t vertexproject/synapse:v2.x.x-py37 -f docker/images/synapse/Dockerfile-py37 .
            docker build -t vertexproject/synapse-aha:v2.x.x -f docker/images/aha/Dockerfile .
            docker build -t vertexproject/synapse-axon:v2.x.x -f docker/images/axon/Dockerfile .
            docker build -t vertexproject/synapse-cortex:v2.x.x -f docker/images/cortex/Dockerfile .
            docker build -t vertexproject/synapse-cryotank:v2.x.x -f docker/images/cryotank/Dockerfile .
      - run:
          name: smoke test image
          command: |
            docker run --rm -it --entrypoint python vertexproject/synapse:v2.x.x -m synapse.servers.cortex --help
            dstatus00=$?
            if [ $dstatus00 != "0" ]; then exit 1; fi
            docker run --rm -it --entrypoint python vertexproject/synapse:v2.x.x-py37 -m synapse.servers.cortex --help
            dstatus01=$?
            if [ $dstatus01 != "0" ]; then exit 1; fi
            docker run -d --name test01 vertexproject/synapse-aha:v2.x.x
            docker run -d --name test02 vertexproject/synapse-axon:v2.x.x
            docker run -d --name test03 vertexproject/synapse-cortex:v2.x.x
            docker run -d --name test04 vertexproject/synapse-cryotank:v2.x.x
      - check_docker_smoketests
      - run:
          name: tag and push container
          command: |
            docker push vertexproject/synapse:v2.x.x
            docker push vertexproject/synapse:v2.x.x-py37
            docker push vertexproject/synapse-aha:v2.x.x
            docker push vertexproject/synapse-axon:v2.x.x
            docker push vertexproject/synapse-cortex:v2.x.x
            docker push vertexproject/synapse-cryotank:v2.x.x

  deploy_docker_branches:
    description: "Deploy current branch images to docker"
    steps:
      - checkout
      - setup_remote_docker
      - do_docker_login
      - run:
          name: install python
          command: |
            apk add python3 py3-pip git
      - run:
          name: setcommit
          command: |
            python3 setup.py setcommit
      - run:
          name: build images
          command: |
            docker build -t vertexproject/synapse:${CIRCLE_BRANCH} -f docker/images/synapse/Dockerfile .
            docker build -t vertexproject/synapse:${CIRCLE_BRANCH}-py37 -f docker/images/synapse/Dockerfile-py37 .
            docker build -t vertexproject/synapse-aha:${CIRCLE_BRANCH} -f docker/images/aha/Dockerfile .
            docker build -t vertexproject/synapse-axon:${CIRCLE_BRANCH} -f docker/images/axon/Dockerfile .
            docker build -t vertexproject/synapse-cortex:${CIRCLE_BRANCH} -f docker/images/cortex/Dockerfile .
            docker build -t vertexproject/synapse-cryotank:${CIRCLE_BRANCH} -f docker/images/cryotank/Dockerfile .
      - run:
          name: smoke test image
          command: |
            docker run --rm -it --entrypoint python vertexproject/synapse:${CIRCLE_BRANCH} -m synapse.servers.cortex --help
            dstatus00=$?
            if [ $dstatus00 != "0" ]; then exit 1; fi
            docker run --rm -it --entrypoint python vertexproject/synapse:${CIRCLE_BRANCH}-py37 -m synapse.servers.cortex --help
            dstatus01=$?
            if [ $dstatus01 != "0" ]; then exit 1; fi
            docker run -d --name test01 vertexproject/synapse-aha:${CIRCLE_BRANCH}
            docker run -d --name test02 vertexproject/synapse-axon:${CIRCLE_BRANCH}
            docker run -d --name test03 vertexproject/synapse-cortex:${CIRCLE_BRANCH}
            docker run -d --name test04 vertexproject/synapse-cryotank:${CIRCLE_BRANCH}
      - check_docker_smoketests
      - run:
          name: tag and push container
          command: |
            docker push vertexproject/synapse:${CIRCLE_BRANCH}
            docker push vertexproject/synapse:${CIRCLE_BRANCH}-py37
            docker push vertexproject/synapse-aha:${CIRCLE_BRANCH}
            docker push vertexproject/synapse-axon:${CIRCLE_BRANCH}
            docker push vertexproject/synapse-cortex:${CIRCLE_BRANCH}
            docker push vertexproject/synapse-cryotank:${CIRCLE_BRANCH}

jobs:

  osx38:
    parallelism: 4
    macos:
      xcode: "12.2.0"

    environment:
      PYVERS: 3.8
      CODECOV_FLAG: osx
      CODECOV_PREFIX: osx
      COVERAGE_ARGS: --cov synapse --no-cov-on-fail

    working_directory: /Users/distiller/repo

    steps:
      - test_steps_osx

  python37:
    parallelism: 8
    docker:
      - image: circleci/python:3.7
        environment:
          PYVERS: 3.7
          RUN_SYNTAX: 1
          CODECOV_FLAG: linux
          SYN_REGRESSION_REPO: ~/git/synapse-regression
          COVERAGE_ARGS: --cov synapse --no-cov-on-fail

    working_directory: ~/repo

    steps:
      - test_steps_python

  python38:
    parallelism: 8
    docker:
      - image: circleci/python:3.8
        environment:
          PYVERS: 3.8
          RUN_SYNTAX: 1
          CODECOV_FLAG: linux
          SYN_REGRESSION_REPO: ~/git/synapse-regression
          COVERAGE_ARGS: --cov synapse --no-cov-on-fail

    working_directory: ~/repo

    steps:
      - test_steps_python

  python37_replay:
    parallelism: 6
    docker:
      - image: circleci/python:3.7
        environment:
          PYVERS: 3.7
          RUN_SYNTAX: 1
          CODECOV_FLAG: linux_replay
          SYN_REGRESSION_REPO: ~/git/synapse-regression
          COVERAGE_ARGS: --cov synapse --no-cov-on-fail
          SYNDEV_NEXUS_REPLAY: 1

    working_directory: ~/repo

    steps:
      - test_steps_python_replay

  python38_replay:
    parallelism: 6
    docker:
      - image: circleci/python:3.8
        environment:
          PYVERS: 3.8
          RUN_SYNTAX: 1
          CODECOV_FLAG: linux_replay
          SYN_REGRESSION_REPO: ~/git/synapse-regression
          COVERAGE_ARGS: --cov synapse --no-cov-on-fail
          SYNDEV_NEXUS_REPLAY: 1

    working_directory: ~/repo

    steps:
      - test_steps_python_replay

  doctests:
    parallelism: 1
    docker:
      - image: circleci/python:3.8
        environment:
          PYVERS: 3.7

    working_directory: ~/repo

    steps:
      - test_steps_doc

  python_package_smoketest:
    docker:
      - image: circleci/python:3.8
        environment:
          PYPI_SMOKE_CODE: import synapse; print(synapse.version)
          PYTHON_TAG: py38

    steps:
      - deploy_pypi_prologue
      - deploy_pypi_wheel_build
      - deploy_pypi_sdist_build

  deploy_pypi:
    docker:
      - image: circleci/python:3.8
        environment:
          PYPI_SMOKE_CODE: import synapse; print(synapse.version)
          PYTHON_TAG: py38

    steps:
      - deploy_pypi_prologue
      - deploy_pypi_tag_verify
      - deploy_pypi_wheel_build
      - deploy_pypi_sdist_build
      - deploy_pypi_upload

  deploy_pypi_37:
    docker:
      - image: circleci/python:3.7
        environment:
          PYPI_SMOKE_CODE: import synapse; print(synapse.version)
          PYTHON_TAG: py37

    steps:
      - deploy_pypi_prologue
      - deploy_pypi_tag_verify
      - deploy_pypi_wheel_build
      - deploy_pypi_upload

  docker_tags:
    docker:
      - image: docker:20.10.1
    steps:
      - deploy_docker_tags

  docker_branches:
    docker:
      - image: docker:20.10.1
    steps:
      - deploy_docker_branches

  docker_tags_2xx:
    docker:
      - image: docker:20.10.1
    steps:
      - deploy_docker_2xx

workflows:
  version: 2
  run_tests:
    jobs:
      - doctests:
          filters:
            tags:
              only: /.*/
            branches:
               only: /.*/

      - osx38:
          filters:
            tags:
              ignore: /.*/
            branches:
              ignore: /.*/

      - python37:
          filters:
            tags:
              only: /.*/
            branches:
              only: /.*/

      - python38:
          filters:
            tags:
              only: /.*/
            branches:
              only: /.*/

      - python37_replay:
          filters:
            tags:
              only: /.*/
            branches:
              only:
                - master

      - python38_replay:
          filters:
            tags:
              only: /.*/
            branches:
              only:
                - master

      - python_package_smoketest:
          filters:
            tags:
              only: /.*/
            branches:
              only:
                - master

      - deploy_pypi:
          requires:
            - doctests
            - python37
            - python38
            - python37_replay
            - python38_replay
            - python_package_smoketest
          context: PublicPypiAccess
          filters:
            tags:
              only: /^v2\.[0-9]+\.[0-9]+((a|b|rc)[0-9]*)?$/
            branches:
              ignore: /.*/

      - deploy_pypi_37:
          requires:
            - doctests
            - python37
            - python38
            - python37_replay
            - python38_replay
            - python_package_smoketest
          context: PublicPypiAccess
          filters:
            tags:
              only: /^v2\.[0-9]+\.[0-9]+((a|b|rc)[0-9]*)?$/
            branches:
              ignore: /.*/

      - docker_branches:
          requires:
            - doctests
            - python37
            - python38
            - python_package_smoketest
          context: SynapseDockerCloudUpload
          filters:
            branches:
              only:
                - master
                - feat_struct_log

      - gh-release/dorelease:
          requires:
            - doctests
            - python37
            - python38
            - python37_replay
            - python38_replay
            - python_package_smoketest
          context: GithubMachine
          filters:
            tags:
              only: /^v2\.[0-9]+\.[0-9]+((a|b|rc)[0-9]*)?$/
            branches:
              ignore: /.*/

      - docker_tags:
          requires:
            - doctests
            - python37
            - python38
            - python37_replay
            - python38_replay
            - python_package_smoketest
          context: SynapseDockerCloudUpload
          filters:
            tags:
              only: /^v2\.[0-9]+\.[0-9]+((a|b|rc)[0-9]*)?$/
            branches:
              ignore: /.*/

      - docker_tags_2xx:
          requires:
            - doctests
            - python37
            - python38
            - python37_replay
            - python38_replay
            - python_package_smoketest
          context: SynapseDockerCloudUpload
          filters:
            tags:
              only: /^v2\.[0-9]+\.[0-9]+((a|b|rc)[0-9]*)?$/
            branches:
              ignore: /.*/

  nightly:
    triggers:
      - schedule:
          cron: "0 12 * * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - doctests
      - python38
      - python38_replay

  weekly:
    triggers:
      - schedule:
          cron: "0 12 1 * *"
          filters:
            branches:
              only:
                - master
    jobs:
      - osx38
      - python37
      - python37_replay