Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
408 lines (348 sloc) 10.3 KB
version: 2.0
filters: &filters
filters:
tags:
only: /^v[0-9]+(\.[0-9]+)*$/
filters__tags: &filters__tags
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+(\.[0-9]+)*$/
workflows:
version: 2
test:
jobs:
- cache:
<<: *filters
- build:
<<: *filters
requires:
- cache
- test--12.0:
<<: *filters
requires:
- cache
- test--11.5:
<<: *filters
requires:
- cache
- test--10.10:
<<: *filters
requires:
- cache
- test--9.6.15:
<<: *filters
requires:
- cache
- test--9.5.19:
<<: *filters
requires:
- cache
- test--9.4.24:
<<: *filters
requires:
- cache
- test--8.4.22:
<<: *filters
requires:
- cache
- test--migrations:
<<: *filters
requires:
- cache
- test--tap-github:
<<: *filters
requires:
- test--12.0
- test--11.5
- test--10.10
- test--9.6.15
- test--9.5.19
- test--9.4.24
- test-release:
<<: *filters__tags
requires:
- test--tap-github
- test--migrations
- build
- approve-release:
<<: *filters__tags
type: approval
requires:
- test-release
- release:
<<: *filters__tags
requires:
- approve-release
cache: &cache deps-v6-{{ checksum "setup.py" }}
restore__cache: &restore__cache
restore_cache:
keys:
- *cache
test__base: &test__base
working_directory: /code/
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Run Tests
command: |
source venv/target-postgres/bin/activate
pytest --verbose tests/unit
environment:
POSTGRES_HOST: "localhost"
POSTGRES_DATABASE: "target_postgres_test"
POSTGRES_USERNAME: "postgres"
- store_artifacts:
path: target/test-results
destination: raw-test-output
jobs:
cache:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
steps:
- checkout
- *restore__cache
- run:
name: Install target-postgres
command: |
python -m venv venv/target-postgres
source venv/target-postgres/bin/activate
pip install -e .[tests]
deactivate
- run:
name: Install older versions of target-postgres for migration testing
command: ./tests/migrations/scripts/install_schema_versions.sh
- run:
name: Install tap-github
command: |
python -m venv venv/tap-github
source venv/tap-github/bin/activate
pip install tap-github
deactivate
- run:
name: Install tap-postgres
command: |
python -m venv venv/tap-postgres
source venv/tap-postgres/bin/activate
pip install tap-postgres
deactivate
- save_cache:
key: *cache
paths:
- "./venv"
- "/usr/local/bin"
- "/usr/local/lib/python3.7/site-packages"
- persist_to_workspace:
root: "./"
paths:
- "./venv"
test--12.0:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:12.0
environment:
POSTGRES_DB: target_postgres_test
test--11.5:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:11.5
environment:
POSTGRES_DB: target_postgres_test
test--10.10:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:10.10
environment:
POSTGRES_DB: target_postgres_test
test--9.6.15:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:9.6.15
environment:
POSTGRES_DB: target_postgres_test
test--9.5.19:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:9.5.19
environment:
POSTGRES_DB: target_postgres_test
test--9.4.24:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:9.4.24
environment:
POSTGRES_DB: target_postgres_test
test--8.4.22:
<<: *test__base
docker:
- image: python:3.7.4-stretch
- image: postgres:8.4.22
environment:
POSTGRES_DB: target_postgres_test
test--tap-github:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
- image: postgres:9.4.24
environment:
POSTGRES_DB: target_postgres_test
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Setup artifacts folder
command: mkdir -p /code/artifacts/data
- run:
name: Setup tap-github
working_directory: /code/.circleci/integration/tap-github
command: sed "s/REPLACE_ME/$TAP_GITHUB_TOKEN/" config-template.json > config.json
- run:
name: Tap -> Data
command: |
source venv/tap-github/bin/activate
cd /code/.circleci/integration/tap-github
tap-github --config config.json --properties properties.json > /code/artifacts/data/tap
deactivate
- run:
name: Data -> Target
command: |
source venv/target-postgres/bin/activate
pip install .
cd /code/.circleci/integration
cat /code/artifacts/data/tap | target-postgres --config target-config.json
deactivate
- run:
name: Target -> Data
command: |
source venv/tap-postgres/bin/activate
cd /code/.circleci/integration/tap-postgres
tap-postgres --config config.json --discover > tmp-properties.json
## Select _every_ table found in properties.
## row-count seems to only show up inside of the necessary metadata object...easier than multi-line-sed
sed 's/"row-count": 0,/"row-count": 0,"selected":true,/g' tmp-properties.json > /code/artifacts/data/properties.json
tap-postgres --config config.json --properties /code/artifacts/data/properties.json > /code/artifacts/data/target
deactivate
- run:
name: Repeatability of Data -> Target
command: |
source venv/target-postgres/bin/activate
pip install .
cd /code/.circleci/integration
cat /code/artifacts/data/tap | target-postgres --config target-config.json
deactivate
cd /code
source venv/tap-postgres/bin/activate
cd /code/.circleci/integration/tap-postgres
tap-postgres --config config.json --discover > tmp-properties.json
## Select _every_ table found in properties.
## row-count seems to only show up inside of the necessary metadata object...easier than multi-line-sed
sed 's/"row-count": 0,/"row-count": 0,"selected":true,/g' tmp-properties.json > /code/artifacts/data/properties.json
tap-postgres --config config.json --properties /code/artifacts/data/properties.json > /code/artifacts/data/target.repeated
deactivate
## TODO: compare repeated data to insure that we only changed _sdc values
# diff /code/artifacts/data/target /code/artifacts/data/target.repeated
- store_artifacts:
path: /code/artifacts
test--migrations:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
- image: postgres:9.4.24
environment:
POSTGRES_DB: target_postgres_test
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Run Tests
command: |
source venv/target-postgres/bin/activate
pytest --verbose tests/migrations
environment:
POSTGRES_HOST: "localhost"
POSTGRES_DATABASE: "target_postgres_test"
POSTGRES_USERNAME: "postgres"
- store_artifacts:
path: /code/tests/migrations/artifacts
destination: raw-test-output
build:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Build distribution
command: |
source venv/target-postgres/bin/activate
pip install --upgrade setuptools wheel twine
python setup.py sdist bdist_wheel
deactivate
- persist_to_workspace:
root: "./"
paths:
- "./dist"
test-release:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Validate tag
command: |
export TAG=`echo $CIRCLE_TAG | sed 's/v//'`
VERSION=`grep version setup.py | sed 's/^.*version="\(.*\)",.*$/\1/'`
echo tag: $TAG equals version: $VERSION '?'
[[ $TAG == $VERSION ]]
- run:
name: Install upload tools
command: pip install --upgrade twine
- run:
name: Test Publish
environment:
TWINE_USERNAME: datamill
TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
command: |
export TWINE_PASSWORD=$PYPI__PASSWORD__TEST
twine upload ./dist/*
release:
working_directory: /code/
docker:
- image: python:3.7.4-stretch
steps:
- checkout
- *restore__cache
- attach_workspace:
at: "./"
- run:
name: Install upload tools
command: pip install --upgrade twine
- run:
name: Publish
environment:
TWINE_USERNAME: datamill
command: |
export TWINE_PASSWORD=$PYPI__PASSWORD
twine upload ./dist/*
You can’t perform that action at this time.