diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..f062d05 Binary files /dev/null and b/.DS_Store differ diff --git a/.circleci/config.yml b/.circleci/config.yml index 50a6f02..1028577 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,30 +11,22 @@ jobs: # specify the version you desire here # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` - image: python:3.7.0-alpine3.8 - environment: - RAILS_ENV: test - PGHOST: 127.0.0.1 - PGUSER: solo # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images # documented at https://circleci.com/docs/2.0/circleci-images/ - - image: postgres:10.5-alpine - environment: - POSTGRES_USER: solo - POSTGRES_DB: solo - - - image: redis:5.0-rc5-alpine steps: - checkout + # https://circleci.com/docs/2.0/building-docker-images/#overview + - setup_remote_docker # Download and cache dependencies - restore_cache: keys: - - v1-dependencies-{{ checksum "requirements.txt" }} - - v1-dependencies-{{ checksum "requirements_local.txt" }} - - v1-dependencies-{{ checksum "requirements_test.txt" }} + - v1-dependencies-{{ checksum "requirements/minimal.txt" }} + - v1-dependencies-{{ checksum "requirements/local.txt" }} + - v1-dependencies-{{ checksum "requirements/test.txt" }} - run: name: install dependencies @@ -44,14 +36,14 @@ jobs: apk update apk add --virtual build-deps gcc make python-dev musl-dev apk add postgresql-dev - pip install -r ./requirements.txt - pip install -r ./requirements_test.txt + pip install -r ./requirements/minimal.txt + pip install -r ./requirements/test.txt pip install coveralls - save_cache: paths: - ./venv - key: v1-dependencies-{{ checksum "requirements.txt" }} + key: v1-dependencies-{{ checksum "requirements/minimal.txt" }} # run tests! # this example uses Django's built-in test-runner diff --git a/.travis.yml b/.travis.yml index 8ffe394..7dc83bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,21 +4,20 @@ sudo: true language: python cache: pip services: - - postgresql - - redis-server -addons: - postgresql: "9.4" + - docker +# addons: +# postgresql: "9.4" install: - - pip install -r ./requirements.txt - - pip install -r ./requirements_test.txt + - pip install -r ./requirements/minimal.txt + - pip install -r ./requirements/test.txt - pip install coveralls python: - "3.7" -before_script: - - psql -c 'create role solo with superuser login;' -U postgres - - psql -c 'create database solo with owner solo;' -U postgres +# before_script: +# - psql -c 'create role solo with superuser login;' -U postgres +# - psql -c 'create database solo with owner solo;' -U postgres # --source specifies what packages to cover, you probably want to use that option script: diff --git a/README.rst b/README.rst index a77fe19..96fd674 100644 --- a/README.rst +++ b/README.rst @@ -63,8 +63,9 @@ You can terminate the server by sending it a SIGINT (Ctrl-C from an interactive Test framework -------------- -Run existing test suite with +You will need docker to run test instances of Postgres and Redis. + +Run existing test suite with: .. code:: - (solo) $ docker-compose -f env/dev/docker-compose.yml up -d (solo) $ py.test diff --git a/Vagrantfile b/Vagrantfile index db31db2..dd9825e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -125,8 +125,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # chef.validation_client_name = "ORGNAME-validator" config.vm.provision :ansible do |ansible| - ansible.playbook = "deploy/develop.yml" - ansible.inventory_path = "deploy/hosts" + ansible.playbook = "env/dev/deploy/develop.yml" + ansible.inventory_path = "env/dev/deploy/hosts" # additional "vv" and "vvv" options are available ansible.verbose = "v" end diff --git a/docs/devdocs/index.rst b/docs/devdocs/index.rst new file mode 100644 index 0000000..ca1e81d --- /dev/null +++ b/docs/devdocs/index.rst @@ -0,0 +1,5 @@ +======================= +Developer Documentation +======================= + + diff --git a/docs/index.rst b/docs/index.rst index 37e3cca..6b52562 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,8 @@ Contents: providers.rst migrations.rst + devdocs/index.rst + Indices and tables diff --git a/env/dev/README.rst b/env/dev/README.rst new file mode 100644 index 0000000..a7ff43b --- /dev/null +++ b/env/dev/README.rst @@ -0,0 +1,5 @@ +Dev Deployment Tools +==================== + + +This directory contains various deployment tools. \ No newline at end of file diff --git a/env/dev/deploy/develop.retry b/env/dev/deploy/develop.retry new file mode 100644 index 0000000..6563189 --- /dev/null +++ b/env/dev/deploy/develop.retry @@ -0,0 +1 @@ +develop diff --git a/deploy/develop.yml b/env/dev/deploy/develop.yml similarity index 100% rename from deploy/develop.yml rename to env/dev/deploy/develop.yml diff --git a/deploy/host_vars/develop b/env/dev/deploy/host_vars/develop similarity index 100% rename from deploy/host_vars/develop rename to env/dev/deploy/host_vars/develop diff --git a/deploy/hosts b/env/dev/deploy/hosts similarity index 100% rename from deploy/hosts rename to env/dev/deploy/hosts diff --git a/deploy/includes/remote_host_prerequisites.yml b/env/dev/deploy/includes/remote_host_prerequisites.yml similarity index 100% rename from deploy/includes/remote_host_prerequisites.yml rename to env/dev/deploy/includes/remote_host_prerequisites.yml diff --git a/deploy/roles/DavidWittman.redis/.gitignore b/env/dev/deploy/roles/DavidWittman.redis/.gitignore similarity index 100% rename from deploy/roles/DavidWittman.redis/.gitignore rename to env/dev/deploy/roles/DavidWittman.redis/.gitignore diff --git a/deploy/roles/DavidWittman.redis/.kitchen.yml b/env/dev/deploy/roles/DavidWittman.redis/.kitchen.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/.kitchen.yml rename to env/dev/deploy/roles/DavidWittman.redis/.kitchen.yml diff --git a/deploy/roles/DavidWittman.redis/.travis.yml b/env/dev/deploy/roles/DavidWittman.redis/.travis.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/.travis.yml rename to env/dev/deploy/roles/DavidWittman.redis/.travis.yml diff --git a/deploy/roles/DavidWittman.redis/Gemfile b/env/dev/deploy/roles/DavidWittman.redis/Gemfile similarity index 100% rename from deploy/roles/DavidWittman.redis/Gemfile rename to env/dev/deploy/roles/DavidWittman.redis/Gemfile diff --git a/deploy/roles/DavidWittman.redis/Gemfile.lock b/env/dev/deploy/roles/DavidWittman.redis/Gemfile.lock similarity index 100% rename from deploy/roles/DavidWittman.redis/Gemfile.lock rename to env/dev/deploy/roles/DavidWittman.redis/Gemfile.lock diff --git a/deploy/roles/DavidWittman.redis/LICENSE b/env/dev/deploy/roles/DavidWittman.redis/LICENSE similarity index 100% rename from deploy/roles/DavidWittman.redis/LICENSE rename to env/dev/deploy/roles/DavidWittman.redis/LICENSE diff --git a/deploy/roles/DavidWittman.redis/README.md b/env/dev/deploy/roles/DavidWittman.redis/README.md similarity index 100% rename from deploy/roles/DavidWittman.redis/README.md rename to env/dev/deploy/roles/DavidWittman.redis/README.md diff --git a/deploy/roles/DavidWittman.redis/defaults/main.yml b/env/dev/deploy/roles/DavidWittman.redis/defaults/main.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/defaults/main.yml rename to env/dev/deploy/roles/DavidWittman.redis/defaults/main.yml diff --git a/deploy/roles/DavidWittman.redis/handlers/main.yml b/env/dev/deploy/roles/DavidWittman.redis/handlers/main.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/handlers/main.yml rename to env/dev/deploy/roles/DavidWittman.redis/handlers/main.yml diff --git a/deploy/roles/DavidWittman.redis/meta/.galaxy_install_info b/env/dev/deploy/roles/DavidWittman.redis/meta/.galaxy_install_info similarity index 100% rename from deploy/roles/DavidWittman.redis/meta/.galaxy_install_info rename to env/dev/deploy/roles/DavidWittman.redis/meta/.galaxy_install_info diff --git a/deploy/roles/DavidWittman.redis/meta/main.yml b/env/dev/deploy/roles/DavidWittman.redis/meta/main.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/meta/main.yml rename to env/dev/deploy/roles/DavidWittman.redis/meta/main.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/check_vars.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/check_vars.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/check_vars.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/check_vars.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/dependencies.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/dependencies.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/dependencies.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/dependencies.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/download.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/download.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/download.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/download.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/install.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/install.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/install.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/install.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/local_facts.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/local_facts.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/local_facts.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/local_facts.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/main.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/main.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/main.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/main.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/sentinel.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/sentinel.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/sentinel.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/sentinel.yml diff --git a/deploy/roles/DavidWittman.redis/tasks/server.yml b/env/dev/deploy/roles/DavidWittman.redis/tasks/server.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/tasks/server.yml rename to env/dev/deploy/roles/DavidWittman.redis/tasks/server.yml diff --git a/deploy/roles/DavidWittman.redis/templates/Debian/redis.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/Debian/redis.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/Debian/redis.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/Debian/redis.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/Debian/redis_sentinel.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/Debian/redis_sentinel.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/Debian/redis_sentinel.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/Debian/redis_sentinel.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/RedHat/redis.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/RedHat/redis.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/RedHat/redis.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/RedHat/redis.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/RedHat/redis_sentinel.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/RedHat/redis_sentinel.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/RedHat/redis_sentinel.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/RedHat/redis_sentinel.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/default/redis.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/default/redis.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/default/redis.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/default/redis.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/default/redis.service.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/default/redis.service.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/default/redis.service.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/default/redis.service.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.init.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.init.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.init.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.init.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.service.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.service.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.service.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/default/redis_sentinel.service.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/etc/ansible/facts.d/redis.fact.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/etc/ansible/facts.d/redis.fact.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/etc/ansible/facts.d/redis.fact.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/etc/ansible/facts.d/redis.fact.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/etc/tmpfiles.d/redis.conf.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/etc/tmpfiles.d/redis.conf.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/etc/tmpfiles.d/redis.conf.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/etc/tmpfiles.d/redis.conf.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/redis.conf.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/redis.conf.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/redis.conf.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/redis.conf.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/redis.init.conf.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/redis.init.conf.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/redis.init.conf.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/redis.init.conf.j2 diff --git a/deploy/roles/DavidWittman.redis/templates/redis_sentinel.conf.j2 b/env/dev/deploy/roles/DavidWittman.redis/templates/redis_sentinel.conf.j2 similarity index 100% rename from deploy/roles/DavidWittman.redis/templates/redis_sentinel.conf.j2 rename to env/dev/deploy/roles/DavidWittman.redis/templates/redis_sentinel.conf.j2 diff --git a/deploy/roles/DavidWittman.redis/test/integration/checksum/default.yml b/env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/default.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/checksum/default.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/default.yml diff --git a/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/redis_spec.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/redis_spec.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/redis_spec.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/redis_spec.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/spec_helper.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/spec_helper.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/spec_helper.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/checksum/serverspec/spec_helper.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/default/default.yml b/env/dev/deploy/roles/DavidWittman.redis/test/integration/default/default.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/default/default.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/default/default.yml diff --git a/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/redis_spec.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/redis_spec.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/default/serverspec/redis_spec.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/redis_spec.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/spec_helper.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/spec_helper.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/default/serverspec/spec_helper.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/default/serverspec/spec_helper.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/logfile/default.yml b/env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/default.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/logfile/default.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/default.yml diff --git a/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/log_spec.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/log_spec.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/log_spec.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/log_spec.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/spec_helper.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/spec_helper.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/spec_helper.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/logfile/serverspec/spec_helper.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/sentinel/default.yml b/env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/default.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/sentinel/default.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/default.yml diff --git a/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/sentinel_spec.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/sentinel_spec.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/sentinel_spec.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/sentinel_spec.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/spec_helper.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/spec_helper.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/spec_helper.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/sentinel/serverspec/spec_helper.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/service-name/default.yml b/env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/default.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/service-name/default.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/default.yml diff --git a/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/redis_spec.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/redis_spec.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/redis_spec.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/redis_spec.rb diff --git a/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/spec_helper.rb b/env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/spec_helper.rb similarity index 100% rename from deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/spec_helper.rb rename to env/dev/deploy/roles/DavidWittman.redis/test/integration/service-name/serverspec/spec_helper.rb diff --git a/deploy/roles/DavidWittman.redis/test/test_all.yml b/env/dev/deploy/roles/DavidWittman.redis/test/test_all.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/test_all.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/test_all.yml diff --git a/deploy/roles/DavidWittman.redis/test/test_sentinel.yml b/env/dev/deploy/roles/DavidWittman.redis/test/test_sentinel.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/test_sentinel.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/test_sentinel.yml diff --git a/deploy/roles/DavidWittman.redis/test/test_server.yml b/env/dev/deploy/roles/DavidWittman.redis/test/test_server.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/test/test_server.yml rename to env/dev/deploy/roles/DavidWittman.redis/test/test_server.yml diff --git a/deploy/roles/DavidWittman.redis/vars/main.yml b/env/dev/deploy/roles/DavidWittman.redis/vars/main.yml similarity index 100% rename from deploy/roles/DavidWittman.redis/vars/main.yml rename to env/dev/deploy/roles/DavidWittman.redis/vars/main.yml diff --git a/deploy/roles/avanov.pyenv/.gitignore b/env/dev/deploy/roles/avanov.pyenv/.gitignore similarity index 100% rename from deploy/roles/avanov.pyenv/.gitignore rename to env/dev/deploy/roles/avanov.pyenv/.gitignore diff --git a/deploy/roles/avanov.pyenv/.travis.yml b/env/dev/deploy/roles/avanov.pyenv/.travis.yml similarity index 100% rename from deploy/roles/avanov.pyenv/.travis.yml rename to env/dev/deploy/roles/avanov.pyenv/.travis.yml diff --git a/deploy/roles/avanov.pyenv/LICENSE b/env/dev/deploy/roles/avanov.pyenv/LICENSE similarity index 100% rename from deploy/roles/avanov.pyenv/LICENSE rename to env/dev/deploy/roles/avanov.pyenv/LICENSE diff --git a/deploy/roles/avanov.pyenv/README.md b/env/dev/deploy/roles/avanov.pyenv/README.md similarity index 100% rename from deploy/roles/avanov.pyenv/README.md rename to env/dev/deploy/roles/avanov.pyenv/README.md diff --git a/deploy/roles/avanov.pyenv/defaults/main.yml b/env/dev/deploy/roles/avanov.pyenv/defaults/main.yml similarity index 100% rename from deploy/roles/avanov.pyenv/defaults/main.yml rename to env/dev/deploy/roles/avanov.pyenv/defaults/main.yml diff --git a/deploy/roles/avanov.pyenv/handlers/main.yml b/env/dev/deploy/roles/avanov.pyenv/handlers/main.yml similarity index 100% rename from deploy/roles/avanov.pyenv/handlers/main.yml rename to env/dev/deploy/roles/avanov.pyenv/handlers/main.yml diff --git a/deploy/roles/avanov.pyenv/meta/.galaxy_install_info b/env/dev/deploy/roles/avanov.pyenv/meta/.galaxy_install_info similarity index 100% rename from deploy/roles/avanov.pyenv/meta/.galaxy_install_info rename to env/dev/deploy/roles/avanov.pyenv/meta/.galaxy_install_info diff --git a/deploy/roles/avanov.pyenv/meta/main.yml b/env/dev/deploy/roles/avanov.pyenv/meta/main.yml similarity index 100% rename from deploy/roles/avanov.pyenv/meta/main.yml rename to env/dev/deploy/roles/avanov.pyenv/meta/main.yml diff --git a/deploy/roles/avanov.pyenv/tasks/Debian.yml b/env/dev/deploy/roles/avanov.pyenv/tasks/Debian.yml similarity index 100% rename from deploy/roles/avanov.pyenv/tasks/Debian.yml rename to env/dev/deploy/roles/avanov.pyenv/tasks/Debian.yml diff --git a/deploy/roles/avanov.pyenv/tasks/RedHat.yml b/env/dev/deploy/roles/avanov.pyenv/tasks/RedHat.yml similarity index 100% rename from deploy/roles/avanov.pyenv/tasks/RedHat.yml rename to env/dev/deploy/roles/avanov.pyenv/tasks/RedHat.yml diff --git a/deploy/roles/avanov.pyenv/tasks/install.yml b/env/dev/deploy/roles/avanov.pyenv/tasks/install.yml similarity index 100% rename from deploy/roles/avanov.pyenv/tasks/install.yml rename to env/dev/deploy/roles/avanov.pyenv/tasks/install.yml diff --git a/deploy/roles/avanov.pyenv/tasks/main.yml b/env/dev/deploy/roles/avanov.pyenv/tasks/main.yml similarity index 100% rename from deploy/roles/avanov.pyenv/tasks/main.yml rename to env/dev/deploy/roles/avanov.pyenv/tasks/main.yml diff --git a/deploy/roles/avanov.pyenv/templates/.pyenvrc.j2 b/env/dev/deploy/roles/avanov.pyenv/templates/.pyenvrc.j2 similarity index 100% rename from deploy/roles/avanov.pyenv/templates/.pyenvrc.j2 rename to env/dev/deploy/roles/avanov.pyenv/templates/.pyenvrc.j2 diff --git a/deploy/roles/avanov.pyenv/test.yml b/env/dev/deploy/roles/avanov.pyenv/test.yml similarity index 100% rename from deploy/roles/avanov.pyenv/test.yml rename to env/dev/deploy/roles/avanov.pyenv/test.yml diff --git a/deploy/roles/avanov.pyenv/vars/main.yml b/env/dev/deploy/roles/avanov.pyenv/vars/main.yml similarity index 100% rename from deploy/roles/avanov.pyenv/vars/main.yml rename to env/dev/deploy/roles/avanov.pyenv/vars/main.yml diff --git a/deploy/roles/locale/defaults/main.yml b/env/dev/deploy/roles/locale/defaults/main.yml similarity index 100% rename from deploy/roles/locale/defaults/main.yml rename to env/dev/deploy/roles/locale/defaults/main.yml diff --git a/deploy/roles/locale/meta/main.yml b/env/dev/deploy/roles/locale/meta/main.yml similarity index 100% rename from deploy/roles/locale/meta/main.yml rename to env/dev/deploy/roles/locale/meta/main.yml diff --git a/deploy/roles/locale/tasks/main.yml b/env/dev/deploy/roles/locale/tasks/main.yml similarity index 100% rename from deploy/roles/locale/tasks/main.yml rename to env/dev/deploy/roles/locale/tasks/main.yml diff --git a/deploy/roles/postgresql/defaults/main.yml b/env/dev/deploy/roles/postgresql/defaults/main.yml similarity index 100% rename from deploy/roles/postgresql/defaults/main.yml rename to env/dev/deploy/roles/postgresql/defaults/main.yml diff --git a/deploy/roles/postgresql/handlers/main.yml b/env/dev/deploy/roles/postgresql/handlers/main.yml similarity index 100% rename from deploy/roles/postgresql/handlers/main.yml rename to env/dev/deploy/roles/postgresql/handlers/main.yml diff --git a/deploy/roles/postgresql/meta/main.yml b/env/dev/deploy/roles/postgresql/meta/main.yml similarity index 100% rename from deploy/roles/postgresql/meta/main.yml rename to env/dev/deploy/roles/postgresql/meta/main.yml diff --git a/deploy/roles/postgresql/tasks/main.yml b/env/dev/deploy/roles/postgresql/tasks/main.yml similarity index 100% rename from deploy/roles/postgresql/tasks/main.yml rename to env/dev/deploy/roles/postgresql/tasks/main.yml diff --git a/deploy/roles/postgresql/templates/.pgpass.j2 b/env/dev/deploy/roles/postgresql/templates/.pgpass.j2 similarity index 100% rename from deploy/roles/postgresql/templates/.pgpass.j2 rename to env/dev/deploy/roles/postgresql/templates/.pgpass.j2 diff --git a/deploy/roles/postgresql/templates/10/main/pg_hba.conf.j2 b/env/dev/deploy/roles/postgresql/templates/10/main/pg_hba.conf.j2 similarity index 100% rename from deploy/roles/postgresql/templates/10/main/pg_hba.conf.j2 rename to env/dev/deploy/roles/postgresql/templates/10/main/pg_hba.conf.j2 diff --git a/deploy/roles/postgresql/templates/10/main/postgresql.conf.j2 b/env/dev/deploy/roles/postgresql/templates/10/main/postgresql.conf.j2 similarity index 100% rename from deploy/roles/postgresql/templates/10/main/postgresql.conf.j2 rename to env/dev/deploy/roles/postgresql/templates/10/main/postgresql.conf.j2 diff --git a/deploy/roles/postgresql/templates/30-postgresql-shm.conf.j2 b/env/dev/deploy/roles/postgresql/templates/30-postgresql-shm.conf.j2 similarity index 100% rename from deploy/roles/postgresql/templates/30-postgresql-shm.conf.j2 rename to env/dev/deploy/roles/postgresql/templates/30-postgresql-shm.conf.j2 diff --git a/deploy/roles/postgresql/templates/pgbouncer.ini.j2 b/env/dev/deploy/roles/postgresql/templates/pgbouncer.ini.j2 similarity index 100% rename from deploy/roles/postgresql/templates/pgbouncer.ini.j2 rename to env/dev/deploy/roles/postgresql/templates/pgbouncer.ini.j2 diff --git a/deploy/roles/postgresql/templates/pgbouncer.j2 b/env/dev/deploy/roles/postgresql/templates/pgbouncer.j2 similarity index 100% rename from deploy/roles/postgresql/templates/pgbouncer.j2 rename to env/dev/deploy/roles/postgresql/templates/pgbouncer.j2 diff --git a/deploy/roles/postgresql/templates/userlist.txt.j2 b/env/dev/deploy/roles/postgresql/templates/userlist.txt.j2 similarity index 100% rename from deploy/roles/postgresql/templates/userlist.txt.j2 rename to env/dev/deploy/roles/postgresql/templates/userlist.txt.j2 diff --git a/deploy/roles/postgresql/vars/main.yml b/env/dev/deploy/roles/postgresql/vars/main.yml similarity index 100% rename from deploy/roles/postgresql/vars/main.yml rename to env/dev/deploy/roles/postgresql/vars/main.yml diff --git a/deploy/roles/site/defaults/main.yml b/env/dev/deploy/roles/site/defaults/main.yml similarity index 100% rename from deploy/roles/site/defaults/main.yml rename to env/dev/deploy/roles/site/defaults/main.yml diff --git a/deploy/roles/site/handlers/main.yml b/env/dev/deploy/roles/site/handlers/main.yml similarity index 100% rename from deploy/roles/site/handlers/main.yml rename to env/dev/deploy/roles/site/handlers/main.yml diff --git a/deploy/roles/site/meta/main.yml b/env/dev/deploy/roles/site/meta/main.yml similarity index 100% rename from deploy/roles/site/meta/main.yml rename to env/dev/deploy/roles/site/meta/main.yml diff --git a/deploy/roles/site/tasks/develop.yml b/env/dev/deploy/roles/site/tasks/develop.yml similarity index 100% rename from deploy/roles/site/tasks/develop.yml rename to env/dev/deploy/roles/site/tasks/develop.yml diff --git a/deploy/roles/site/tasks/main.yml b/env/dev/deploy/roles/site/tasks/main.yml similarity index 100% rename from deploy/roles/site/tasks/main.yml rename to env/dev/deploy/roles/site/tasks/main.yml diff --git a/deploy/roles/site/vars/main.yml b/env/dev/deploy/roles/site/vars/main.yml similarity index 100% rename from deploy/roles/site/vars/main.yml rename to env/dev/deploy/roles/site/vars/main.yml diff --git a/env/dev/docker-compose.yml b/env/dev/docker-compose.yml index 196b71d..757ae4e 100644 --- a/env/dev/docker-compose.yml +++ b/env/dev/docker-compose.yml @@ -1,7 +1,7 @@ version: "2" services: postgres: - image: postgres:10.5-alpine + image: postgres:11.2-alpine environment: POSTGRES_USER: solo POSTGRES_PASSWORD: solo @@ -10,6 +10,6 @@ services: - 5432:5432 redis: - image: redis:5.0-rc5-alpine + image: redis:5.0.4-alpine ports: - 6379:6379 diff --git a/requirements_local.txt b/requirements/local.txt similarity index 100% rename from requirements_local.txt rename to requirements/local.txt diff --git a/requirements.txt b/requirements/minimal.txt similarity index 90% rename from requirements.txt rename to requirements/minimal.txt index bbf142d..757fdb3 100644 --- a/requirements.txt +++ b/requirements/minimal.txt @@ -1,3 +1,4 @@ +uvicorn==0.6.1 aiohttp==3.5.4 aiohttp_session[aioredis]==2.7.0 aiopg==0.16.0 @@ -12,5 +13,5 @@ venusian==1.2.0 uvloop>=0.12.1,<0.13.0 # for the parser inflection==0.3.1 -typeit==0.8.0 +typeit==0.9.0 routes==2.4.1 diff --git a/requirements_test.txt b/requirements/test.txt similarity index 54% rename from requirements_test.txt rename to requirements/test.txt index 47ad56c..ab9d4ba 100644 --- a/requirements_test.txt +++ b/requirements/test.txt @@ -2,4 +2,7 @@ pytest>=4.3.1,<4.4 pytest-aiohttp==0.3.0 coverage>=4.5.3,<4.6 pytest-cov==2.6.1 -mypy==0.670 \ No newline at end of file +mypy==0.670 +docker>=3.7.1,<3.8 +faker>=1.0.4,<1.1 +redis>=3.2.1,<3.3 \ No newline at end of file diff --git a/setup.py b/setup.py index 2084508..a82786b 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,14 @@ -import os - from pathlib import Path from setuptools import setup from setuptools import find_packages -here = Path(os.path.abspath(os.path.dirname(__file__))) +HERE = Path(__file__).absolute().parent -with here.joinpath('README.rst').open() as f: +with (HERE / 'README.rst').open() as f: README = f.read() -with here.joinpath('requirements.txt').open() as f: +with (HERE / 'requirements' / 'minimal.txt').open() as f: rows = f.read().strip().split('\n') requires = [] for row in rows: diff --git a/solo/apps/accounts/__init__.py b/solo/apps/accounts/__init__.py index 870ae14..51f2b05 100644 --- a/solo/apps/accounts/__init__.py +++ b/solo/apps/accounts/__init__.py @@ -5,7 +5,7 @@ from . import predicate -__all__ = ['get_user'] +__all__ = ['get_user'] def includeme(config: Configurator): diff --git a/solo/cli/__init__.py b/solo/cli/__init__.py index b908f71..80ba7aa 100644 --- a/solo/cli/__init__.py +++ b/solo/cli/__init__.py @@ -5,12 +5,13 @@ import logging import logging.config import sys +from pathlib import Path from pkg_resources import get_distribution -from solo.cli import run +from . import commands from solo.integrations.alembic import integrate_alembic_cli -from solo.server.config import Config, EventLoopType +from solo.config.app import Config, EventLoopType from .util import parse_app_config @@ -27,11 +28,11 @@ def main(args=None, stdout=None): # $ solo run # --------------------------- - run.setup(subparsers) + commands.run.setup(subparsers) # $ solo db [args] # --------------------------- - integrate_alembic_cli(subparsers) + integrate_alembic_cli(subparsers, prefix='db') # Common arguments # ---------------- @@ -42,7 +43,7 @@ def main(args=None, stdout=None): if args is None: args = sys.argv[1:] args = parser.parse_args(args) - solo_cfg = parse_app_config(args.solocfg) + solo_cfg = parse_app_config(Path(args.solocfg)) # Set up and run # -------------- diff --git a/solo/cli/commands/__init__.py b/solo/cli/commands/__init__.py new file mode 100644 index 0000000..b512710 --- /dev/null +++ b/solo/cli/commands/__init__.py @@ -0,0 +1,4 @@ +from . import run + + +__all__ = ('run',) \ No newline at end of file diff --git a/solo/cli/run.py b/solo/cli/commands/run.py similarity index 95% rename from solo/cli/run.py rename to solo/cli/commands/run.py index 0854608..e57b2ab 100644 --- a/solo/cli/run.py +++ b/solo/cli/commands/run.py @@ -4,7 +4,7 @@ import sys from solo import init_webapp -from solo.server.config import Config +from solo.config.app import Config def setup(subparsers): diff --git a/solo/cli/util.py b/solo/cli/util.py index 1ddeb21..22ff7f0 100644 --- a/solo/cli/util.py +++ b/solo/cli/util.py @@ -1,11 +1,18 @@ from pathlib import Path -from solo.server import config +from solo.config import app +from solo.config import docker_compose import yaml -def parse_app_config(path: str) -> config.Config: - with Path(path).open('r') as f: - cfg = yaml.load(f.read()) - return config.MakeConfig(cfg) \ No newline at end of file +def parse_app_config(path: Path) -> app.Config: + with path.open('r') as f: + cfg = yaml.load(f.read(), Loader=yaml.FullLoader) + return app.mk_config(cfg) + + +def parse_compose_config(path: Path) -> docker_compose.Config: + with path.open('r') as f: + cfg = yaml.load(f.read(), Loader=yaml.FullLoader) + return docker_compose.mk_config(cfg) diff --git a/solo/config/__init__.py b/solo/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/solo/server/config.py b/solo/config/app.py similarity index 74% rename from solo/server/config.py rename to solo/config/app.py index fc57729..282c010 100644 --- a/solo/server/config.py +++ b/solo/config/app.py @@ -25,9 +25,9 @@ class Redis(NamedTuple): class Postgresql(NamedTuple): - user: str - dbname: str - password: str + user: str = 'solo' + dbname: str = 'solo' + password: str = 'solo' host: str = '127.0.0.1' port: int = 5432 min_connections: int = 1 @@ -48,14 +48,21 @@ class Server(NamedTuple): event_loop: EventLoopType = EventLoopType.ASYNCIO +class Testing(NamedTuple): + docker_pull: bool = True + """ Pull images from registry if they are not available locally yet + """ + + class Config(NamedTuple): server: Server - postgresql: Postgresql - redis: Redis session: Session apps: Sequence[AppConfig] = [] logging: Dict = {'version': 1} debug: bool = True + postgresql: Postgresql = Postgresql() + redis: Redis = Redis() + testing: Testing = Testing() -MakeConfig, serializer = type_constructor(Config) \ No newline at end of file +mk_config, dict_config = type_constructor(Config) \ No newline at end of file diff --git a/solo/config/docker_compose.py b/solo/config/docker_compose.py new file mode 100644 index 0000000..0f37d95 --- /dev/null +++ b/solo/config/docker_compose.py @@ -0,0 +1,94 @@ +from typing import NamedTuple, Optional, Sequence + +import colander +from typeit import type_constructor +from typeit.definitions import TypeExtension +from typeit.schema import Str + + +class _ContainerPortSchema(Str): + def deserialize(self, node, cstruct): + rv = super().deserialize(node, cstruct) + if isinstance(rv, str): + try: + host_port, container_port = rv.split(':') + except ValueError: + container_port = rv + host_port = None + return ContainerPort( + host_port=host_port, + container_port=container_port + ) + return rv + + +class _DockerImageSchema(Str): + def deserialize(self, node, cstruct): + rv = super().deserialize(node, cstruct) + if isinstance(rv, str): + try: + name, tag = rv.split(':') + except ValueError: + name = rv + tag = 'latest' + return DockerImage( + name=name, + tag=tag + ) + return rv + + +class ContainerPort(NamedTuple): + host_port: Optional[str] + container_port: str + + +class DockerImage(NamedTuple): + name: str + tag: str = 'latest' + + @property + def full_name(self) -> str: + return f'{self.name}:{self.tag}' + + +class ServicesRedis(NamedTuple): + image: DockerImage + ports: Sequence[ContainerPort] + + +class ServicesPostgresEnvironment(NamedTuple): + POSTGRES_USER: str + POSTGRES_PASSWORD: str + POSTGRES_DB: str + + +class ServicesPostgres(NamedTuple): + image: DockerImage + environment: ServicesPostgresEnvironment + ports: Sequence[ContainerPort] + + +class Services(NamedTuple): + postgres: ServicesPostgres + redis: ServicesRedis + + +class Config(NamedTuple): + version: str + services: Services + + +mk_config, dict_config = type_constructor( + Config, + overrides={ + ContainerPort: TypeExtension( + schema=colander.SchemaNode(_ContainerPortSchema()) + ), + DockerImage: TypeExtension( + schema=colander.SchemaNode( + _DockerImageSchema() + ) + ) + } +) diff --git a/solo/configurator/__init__.py b/solo/configurator/__init__.py index 8cff591..cedf0b6 100644 --- a/solo/configurator/__init__.py +++ b/solo/configurator/__init__.py @@ -11,7 +11,8 @@ import ramlfications as raml import venusian -from ..server.config import Config +from solo.server.csrf import SessionCSRFStoragePolicy +from ..config.app import Config from .util import maybe_dotted from .config.rendering import BUILTIN_RENDERERS from .config.rendering import RenderingConfigurator @@ -19,6 +20,7 @@ from .config.views import ViewsConfigurator from .config.sums import SumTypesConfigurator from .exceptions import ConfigurationError +from .registry import Registry from .path import caller_package from .view import http_defaults from .view import http_endpoint @@ -88,24 +90,22 @@ class Configurator: def __init__(self, app: Application, + config: Config, route_prefix=None, - registry=None, router_configurator=RoutesConfigurator, views_configurator=ViewsConfigurator, rendering_configurator=RenderingConfigurator, sum_types_configurator=SumTypesConfigurator) -> None: if route_prefix is None: route_prefix = '' - if registry is None: - registry = {} - self.app = app + self.registry = Registry(config=config, + csrf_policy=SessionCSRFStoragePolicy()) self.router = router_configurator(app, route_prefix) self.views = views_configurator(app) self.rendering = rendering_configurator(app) self.sums = sum_types_configurator(app) self._directives = {} - self.registry = registry self.setup_configurator() def include(self, callable, route_prefix: Optional[str] = None): diff --git a/solo/configurator/registry.py b/solo/configurator/registry.py index 1c84282..982fd37 100644 --- a/solo/configurator/registry.py +++ b/solo/configurator/registry.py @@ -1,5 +1,17 @@ +from typing import NamedTuple, Dict + +from ..config.app import Config +from ..server.csrf import SessionCSRFStoragePolicy + + class predvalseq(tuple): """ This class is a copy of ``pyramid.registry.predvalseq`` A subtype of tuple used to represent a sequence of predicate values """ pass + + +class Registry(NamedTuple): + config: Config + csrf_policy: SessionCSRFStoragePolicy + settings: Dict = {} \ No newline at end of file diff --git a/solo/integrations/alembic/cli.py b/solo/integrations/alembic/cli.py index b9cc95b..9ad6bab 100644 --- a/solo/integrations/alembic/cli.py +++ b/solo/integrations/alembic/cli.py @@ -130,7 +130,7 @@ } -def integrate_alembic_cli(parent_cli: argparse._SubParsersAction, prefix: str = 'db'): +def integrate_alembic_cli(parent_cli: argparse._SubParsersAction, prefix: str = 'db') -> None: """ """ db = parent_cli.add_parser(prefix, help='Database commands integrated with Alembic CLI') diff --git a/solo/server/csrf.py b/solo/server/csrf.py new file mode 100644 index 0000000..9bd95b2 --- /dev/null +++ b/solo/server/csrf.py @@ -0,0 +1,236 @@ +import uuid + +from aiohttp_session import get_session + + +class SessionCSRFStoragePolicy: + """ Taken from + https://github.com/Pylons/pyramid/blob/3ee04cc62205b10eb9041b0df5e156936765202f/pyramid/csrf.py#L57 + + A CSRF storage policy that persists the CSRF token in the session. + + Note that using this CSRF implementation requires that + a :term:`session factory` is configured. + + ``key`` + + The session key where the CSRF token will be stored. + Default: `_csrft_`. + + """ + _token_factory = staticmethod(lambda: uuid.uuid4().hex) + + def __init__(self, key: str = '_csrft_') -> None: + self.key = key + + async def new_csrf_token(self, request): + """ Sets a new CSRF token into the session and returns it. """ + session = await get_session(request) + token = self._token_factory() + session[self.key] = token + return token + + async def get_csrf_token(self, request): + """ Returns the currently active CSRF token from the session, + generating a new one if needed.""" + session = await get_session(request) + token = session.get(self.key, None) + if not token: + token = await self.new_csrf_token(request) + return token + + async def check_csrf_token(self, request, supplied_token): + """ Returns ``True`` if the ``supplied_token`` is valid.""" + expected_token = await self.get_csrf_token(request) + return expected_token == supplied_token + + +def get_csrf_token(request): + """ Taken from + https://github.com/Pylons/pyramid/blob/3ee04cc62205b10eb9041b0df5e156936765202f/pyramid/csrf.py#L159 + + Get the currently active CSRF token for the request passed, generating + a new one using ``new_csrf_token(request)`` if one does not exist. This + calls the equivalent method in the chosen CSRF protection implementation. + + .. versionadded :: 1.9 + + """ + registry = request.registry + csrf = registry.getUtility(ICSRFStoragePolicy) + return csrf.get_csrf_token(request) + + +def new_csrf_token(request): + """ Taken from + https://github.com/Pylons/pyramid/blob/3ee04cc62205b10eb9041b0df5e156936765202f/pyramid/csrf.py#L172 + + Generate a new CSRF token for the request passed and persist it in an + implementation defined manner. This calls the equivalent method in the + chosen CSRF protection implementation. + + .. versionadded :: 1.9 + + """ + registry = request.registry + csrf = registry.getUtility(ICSRFStoragePolicy) + return csrf.new_csrf_token(request) + + +def check_csrf_token(request, + token='csrf_token', + header='X-CSRF-Token', + raises=True): + """ Taken from + https://github.com/Pylons/pyramid/blob/3ee04cc62205b10eb9041b0df5e156936765202f/pyramid/csrf.py#L185 + + Check the CSRF token returned by the + :class:`pyramid.interfaces.ICSRFStoragePolicy` implementation against the + value in ``request.POST.get(token)`` (if a POST request) or + ``request.headers.get(header)``. If a ``token`` keyword is not supplied to + this function, the string ``csrf_token`` will be used to look up the token + in ``request.POST``. If a ``header`` keyword is not supplied to this + function, the string ``X-CSRF-Token`` will be used to look up the token in + ``request.headers``. + + If the value supplied by post or by header cannot be verified by the + :class:`pyramid.interfaces.ICSRFStoragePolicy`, and ``raises`` is + ``True``, this function will raise an + :exc:`pyramid.exceptions.BadCSRFToken` exception. If the values differ + and ``raises`` is ``False``, this function will return ``False``. If the + CSRF check is successful, this function will return ``True`` + unconditionally. + + See :ref:`auto_csrf_checking` for information about how to secure your + application automatically against CSRF attacks. + + .. versionadded:: 1.4a2 + + .. versionchanged:: 1.7a1 + A CSRF token passed in the query string of the request is no longer + considered valid. It must be passed in either the request body or + a header. + + .. versionchanged:: 1.9 + Moved from :mod:`pyramid.session` to :mod:`pyramid.csrf` and updated + to use the configured :class:`pyramid.interfaces.ICSRFStoragePolicy` to + verify the CSRF token. + + """ + supplied_token = "" + # We first check the headers for a csrf token, as that is significantly + # cheaper than checking the POST body + if header is not None: + supplied_token = request.headers.get(header, "") + + # If this is a POST/PUT/etc request, then we'll check the body to see if it + # has a token. We explicitly use request.POST here because CSRF tokens + # should never appear in an URL as doing so is a security issue. We also + # explicitly check for request.POST here as we do not support sending form + # encoded data over anything but a request.POST. + if supplied_token == "" and token is not None: + supplied_token = request.POST.get(token, "") + + policy = request.registry.getUtility(ICSRFStoragePolicy) + if not policy.check_csrf_token(request, text_(supplied_token)): + if raises: + raise BadCSRFToken('check_csrf_token(): Invalid token') + return False + return True + + +def check_csrf_origin(request, trusted_origins=None, raises=True): + """ Taken from + https://github.com/Pylons/pyramid/blob/3ee04cc62205b10eb9041b0df5e156936765202f/pyramid/csrf.py#L244 + + Check the ``Origin`` of the request to see if it is a cross site request or + not. + + If the value supplied by the ``Origin`` or ``Referer`` header isn't one of the + trusted origins and ``raises`` is ``True``, this function will raise a + :exc:`pyramid.exceptions.BadCSRFOrigin` exception, but if ``raises`` is + ``False``, this function will return ``False`` instead. If the CSRF origin + checks are successful this function will return ``True`` unconditionally. + + Additional trusted origins may be added by passing a list of domain (and + ports if non-standard like ``['example.com', 'dev.example.com:8080']``) in + with the ``trusted_origins`` parameter. If ``trusted_origins`` is ``None`` + (the default) this list of additional domains will be pulled from the + ``pyramid.csrf_trusted_origins`` setting. + + Note that this function will do nothing if ``request.scheme`` is not + ``https``. + + .. versionadded:: 1.7 + + .. versionchanged:: 1.9 + Moved from :mod:`pyramid.session` to :mod:`pyramid.csrf` + + """ + def _fail(reason): + if raises: + raise BadCSRFOrigin(reason) + else: + return False + + if request.scheme == "https": + # Suppose user visits http://example.com/ + # An active network attacker (man-in-the-middle, MITM) sends a + # POST form that targets https://example.com/detonate-bomb/ and + # submits it via JavaScript. + # + # The attacker will need to provide a CSRF cookie and token, but + # that's no problem for a MITM when we cannot make any assumptions + # about what kind of session storage is being used. So the MITM can + # circumvent the CSRF protection. This is true for any HTTP connection, + # but anyone using HTTPS expects better! For this reason, for + # https://example.com/ we need additional protection that treats + # http://example.com/ as completely untrusted. Under HTTPS, + # Barth et al. found that the Referer header is missing for + # same-domain requests in only about 0.2% of cases or less, so + # we can use strict Referer checking. + + # Determine the origin of this request + origin = request.headers.get("Origin") + if origin is None: + origin = request.referrer + + # Fail if we were not able to locate an origin at all + if not origin: + return _fail("Origin checking failed - no Origin or Referer.") + + # Parse our origin so we we can extract the required information from + # it. + originp = urlparse.urlparse(origin) + + # Ensure that our Referer is also secure. + if originp.scheme != "https": + return _fail( + "Referer checking failed - Referer is insecure while host is " + "secure." + ) + + # Determine which origins we trust, which by default will include the + # current origin. + if trusted_origins is None: + trusted_origins = aslist( + request.registry.settings.get( + "pyramid.csrf_trusted_origins", []) + ) + + if request.host_port not in set(["80", "443"]): + trusted_origins.append("{0.domain}:{0.host_port}".format(request)) + else: + trusted_origins.append(request.domain) + + # Actually check to see if the request's origin matches any of our + # trusted origins. + if not any(is_same_domain(originp.netloc, host) + for host in trusted_origins): + reason = ( + "Referer checking failed - {0} does not match any trusted " + "origins." + ) + return _fail(reason.format(origin)) + + return True diff --git a/solo/server/db/__init__.py b/solo/server/db/__init__.py index 46d80e7..7a9a7de 100644 --- a/solo/server/db/__init__.py +++ b/solo/server/db/__init__.py @@ -3,7 +3,7 @@ import aiopg.sa from solo.configurator.exceptions import ConfigurationError -from solo.server.config import Config +from solo.config.app import Config log = logging.getLogger(__name__) diff --git a/solo/server/memstore.py b/solo/server/memstore.py index a84ef89..f9e3782 100644 --- a/solo/server/memstore.py +++ b/solo/server/memstore.py @@ -2,7 +2,7 @@ import aioredis from aioredis.abc import AbcPool -from .config import Config +from ..config.app import Config async def init_pool(loop: asyncio.AbstractEventLoop, diff --git a/solo/server/startup.py b/solo/server/startup.py index 7372fd2..3361bc4 100644 --- a/solo/server/startup.py +++ b/solo/server/startup.py @@ -1,6 +1,5 @@ import asyncio import logging -from typing import NamedTuple, Dict import aiohttp_session from aiohttp import web @@ -9,27 +8,23 @@ from solo import Configurator from solo.configurator import ApplicationManager from solo.configurator.exceptions import ConfigurationError +from solo.configurator.registry import Registry from solo.configurator.url import complete_route_pattern from solo.configurator.view import PredicatedHandler from solo.server import db from solo.server import memstore -from solo.server.config import Config +from solo.config.app import Config log = logging.getLogger(__name__) -class Registry(NamedTuple): - config: Config - settings: Dict = {} - async def init_webapp(loop: asyncio.AbstractEventLoop, config: Config) -> ApplicationManager: webapp = web.Application(loop=loop, debug=config.debug) - registry = Registry(config=config) - configurator = Configurator(webapp, registry=registry) + configurator = Configurator(webapp, config=config) for app in config.apps: log.debug("------- Setting up {} -------".format(app.name)) @@ -71,7 +66,7 @@ def register_routes(namespace: str, webapp: web.Application, configurator: Confi # Setup routes # ------------ application_routes = configurator.router.routes[namespace] - for route in application_routes.values(): # type: Route + for route in application_routes.values(): handler = PredicatedHandler(route.rules, route.view_metas) guarded_route_pattern = complete_route_pattern(route.pattern, route.rules) log.debug('Binding route {} to the handler named {} in the namespace {}.'.format( diff --git a/solo/server/uviapp.py b/solo/server/uviapp.py new file mode 100644 index 0000000..65393d9 --- /dev/null +++ b/solo/server/uviapp.py @@ -0,0 +1,27 @@ +import routes + + +route_map = routes.Mapper() +route_map.connect("home", "/", controller=object(), action="index", blah=True) +route_map.connect("home", "/users", controller="users", action="users") + + +class App: + def __init__(self, scope): + print('Scope', scope) + self.scope = scope + + async def __call__(self, receive, send): + result = route_map.match(self.scope['path']) + print(result) + await send({ + 'type': 'http.response.start', + 'status': 200, + 'headers': [ + [b'content-type', b'text/plain'], + ] + }) + await send({ + 'type': 'http.response.body', + 'body': b'Hello, from uviapp!', + }) diff --git a/test_config.yml b/test_config.yml index 6cc2ead..a3e8c3f 100644 --- a/test_config.yml +++ b/test_config.yml @@ -40,6 +40,10 @@ apps: - "user:email" - public_repo +testing: + docker_pull: true + + logging: version: 1 formatters: diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..a13ec59 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +from pathlib import Path + +TESTS_ROOT = Path(__file__).absolute().parent diff --git a/tests/conftest.py b/tests/conftest.py index c282b52..7b546d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,18 +1,229 @@ +import logging import pytest +import time +import socket +import psycopg2 +import redis from aiohttp import web +from faker import Faker +from docker.api import APIClient as DockerAPIClient +from docker import from_env as docker_from_env from solo.cli.util import parse_app_config +from solo.cli.util import parse_compose_config from solo import init_webapp +from solo.config.app import Config as AppConfig +from solo.config import docker_compose as compose_config +from . import TESTS_ROOT -@pytest.fixture -def app_config(): - config = parse_app_config('./test_config.yml') # TODO: move path to outer scope - return config +@pytest.fixture(scope='session') +def testing_session_id(): + """ Something that we can use to identify the session, + and is more readable than uuid. + """ + fake = Faker() + return fake.slug() + + +@pytest.fixture(scope='session') +def logger(app_config): + logging.config.dictConfig(app_config.logging) + return logging.getLogger('solo-tests') + + +@pytest.fixture(scope='session') +def app_config() -> AppConfig: + # TODO: move path to outer scope + cfg_path = TESTS_ROOT.parent / 'test_config.yml' + config = parse_app_config(cfg_path) + yield config + + +@pytest.fixture(scope='session') +def docker_compose_config() -> compose_config.Config: + cfg_path = TESTS_ROOT.parent / 'env' / 'dev' / 'docker-compose.yml' + config = parse_compose_config(cfg_path) + yield config + + +@pytest.fixture(scope='session') +def docker_client() -> DockerAPIClient: + return docker_from_env().api @pytest.fixture -def web_client(loop, test_client, app_config): +def web_client( + loop, + test_client, + app_config, + pg_server, + redis_server, +): app_manager = loop.run_until_complete(init_webapp(loop, app_config)) app = app_manager.app return loop.run_until_complete(test_client(app)) + + +@pytest.fixture(scope='session') +def unused_host_port(): + def port_selector(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('127.0.0.1', 0)) + return s.getsockname()[1] + + return port_selector + + +@pytest.fixture(scope='session') +def pg_server( + docker_client, + app_config, + docker_compose_config, + testing_session_id, + unused_host_port, + logger, +): + logger.info('Attempting to start Postgresql container') + + service = docker_compose_config.services.postgres + + if app_config.testing.docker_pull: + image = service.image + docker_client.pull(image.full_name) + + pg_ports = service.ports + container_args = dict( + image=service.image.full_name, + name=f'solo-pg-test-server-{testing_session_id}', + ports=[ + p.host_port and f'{p.host_port}:{p.container_port}' or p.container_port + for p in pg_ports + ], + environment=service.environment._asdict(), + detach=True, + ) + + host = "127.0.0.1" + host_port = pg_ports[0].host_port or unused_host_port() + container_port = pg_ports[0].container_port + port_bindings = {container_port: (host, host_port)} + + logger.info(f'Linking containerized Postgres on port {container_port} ' + f'to {host}:{host_port}') + + container_args['host_config'] = docker_client.create_host_config( + port_bindings=port_bindings + ) + + container = docker_client.create_container(**container_args) + + try: + docker_client.start(container=container['Id']) + server_params = dict( + database=service.environment.POSTGRES_DB, + user=service.environment.POSTGRES_USER, + password=service.environment.POSTGRES_PASSWORD, + host=host, + port=host_port + ) + delay = 0.01 + for i in range(100): + logger.info('Attempting to connect to the containerized Postgres...') + try: + with psycopg2.connect(**server_params) as conn: + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION intarray;") + break + except psycopg2.Error: + time.sleep(delay) + delay *= 2 + else: + pytest.fail( + 'Unable to initialize Postgresql from docker-compose' + ) + + container['host'] = host + container['port'] = host_port + container['pg_params'] = server_params + + yield container + + finally: + docker_client.kill(container=container['Id']) + docker_client.remove_container(container['Id']) + + +@pytest.fixture(scope='session') +def redis_server( + docker_client, + app_config, + docker_compose_config, + testing_session_id, + unused_host_port, + logger, +): + logger.info('Attempting to start Redis container') + + service = docker_compose_config.services.redis + + if app_config.testing.docker_pull: + docker_client.pull( + service.image.full_name + ) + + redis_ports = service.ports + container_args = dict( + image=service.image.full_name, + name=f'solo-redis-test-server-{testing_session_id}', + ports=[ + p.host_port and f'{p.host_port}:{p.container_port}' or p.container_port + for p in redis_ports + ], + detach=True, + ) + + host = "127.0.0.1" + host_port = redis_ports[0].host_port or unused_host_port() + container_port = redis_ports[0].container_port + port_bindings = {container_port: (host, host_port)} + + logger.info(f'Linking containerized Redis on port {container_port} ' + f'to {host}:{host_port}') + + container_args['host_config'] = docker_client.create_host_config( + port_bindings=port_bindings + ) + + container = docker_client.create_container(**container_args) + + try: + docker_client.start(container=container['Id']) + server_params = dict( + host=host, + port=host_port, + ) + delay = 0.01 + for i in range(100): + logger.info('Attempting to connect to the containerized Redis...') + try: + r = redis.Redis(**server_params) + r.ping() + break + except redis.exceptions.ConnectionError: + time.sleep(delay) + delay *= 2 + else: + pytest.fail( + 'Unable to initialize Redis from docker-compose' + ) + + container['host'] = host + container['port'] = host_port + container['redis_params'] = server_params + + yield container + + finally: + docker_client.kill(container=container['Id']) + docker_client.remove_container(container['Id']) diff --git a/tests/integration/apps/auth/api/test_handlers.py b/tests/integration/apps/auth/api/test_handlers.py index eb952fe..112d9c5 100644 --- a/tests/integration/apps/auth/api/test_handlers.py +++ b/tests/integration/apps/auth/api/test_handlers.py @@ -2,7 +2,7 @@ async def test_url__login__be__github(web_client): resp = await web_client.post("/api/login/github", allow_redirects=False) assert resp.status == 302 + async def test_url__login__be__github__callback(web_client): resp = await web_client.get("/api/login/github/callback", allow_redirects=False) assert resp.status == 400 -