Merge pull request #7099 from EricSoroos/local-testing

Local Testing Infrastructure

.circleci/config.yml

@@ -2,30 +2,18 @@ version: 2
 defaults:
   init_environemnt: &init_environment
     run: |
-      # Database Creation
-      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
-      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
-
-      # Database Initialization
-      ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
-      psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;"
-      ckan -c test-core-circle-ci.ini db init
-      gunzip .test_durations.gz
+      test-infrastructure/init_environment.sh
   install_deps: &install_deps
     run: |
-      # OS Dependencies
-      apt update
-      apt install -y postgresql-client
+      test-infrastructure/install_deps.sh
   run_tests: &run_tests
     # Tests Backend, split across containers by segments
     run: |
       mkdir -p ~/junit/result
       python -m pytest ${PYTEST_COMMON_OPTIONS} --splits 4 --group $((CIRCLE_NODE_INDEX+1)) --splitting-algorithm least_duration
 
   ckan_env: &ckan_env
+    # Note, this should be in sync with test-infrastructure/docker-compose.yml
     environment:
       CKAN_DATASTORE_POSTGRES_DB: datastore_test
       CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
@@ -40,17 +28,20 @@ defaults:
       # report usage of deprecated features
       SQLALCHEMY_WARN_20: 1
   pg_image: &pg_image
+    # Note, this should be in sync with test-infrastructure/docker-compose.yml
     image: postgres:10
     environment:
       POSTGRES_USER: ckan
       POSTGRES_PASSWORD: ckan
     name: ckan-postgres
 
   redis_image: &redis_image
+    # Note, this should be in sync with test-infrastructure/docker-compose.yml
     image: redis:3
     name: ckan-redis
 
   solr_image: &solr_image
+    # Note, this should be in sync with test-infrastructure/docker-compose.yml
     image: ckan/ckan-solr:master
     name: ckan-solr
 
@@ -69,12 +60,6 @@ jobs:
       - checkout
 
       - <<: *install_deps
-      - run: |
-          # Python Dependencies
-          pip install -r requirements.txt
-          pip install -r dev-requirements.txt
-          python setup.py develop
-          pip check
       - <<: *init_environment
 
       # Tests Backend, split across containers by segments

.gitignore

@@ -52,3 +52,12 @@ cypress.env.json
 
 # cypress
 cypress/videos
+
+#testing
+junit
+.coverage*
+.cache
+.bash_history
+.gitconfig
+.npm
+.test_durations

changes/7099.changes

@@ -0,0 +1 @@
+Refactor CircleCI testing to allow for local test runs in docker with the same images as CircleCI

doc/contributing/test.rst

@@ -20,12 +20,62 @@ tested manually.
 Back-end tests
 --------------
 
-Most of CKAN's testsuite is for the backend Python code.
+Most of CKAN's testsuite is for the backend Python code. You can run
+the code in a dockerized environment that replicates circleci, or you
+can use a virtual environment based testing.
+
+~~~~~~~~~~~~~~~~
+Dockerized Tests
+~~~~~~~~~~~~~~~~
+
+The ``test-infrastructure`` directory contains a configuration using
+docker compose replicating the circleci test process on the local
+machine.
+
+Set up the testing environment
+==============================
+.. parsed-literal::
+
+   cd test-infrastructure
+   ./setup.sh
+
+This starts a docker compose environment with the supporting postgres,
+redis, and solr containers from the circleci test environment. The
+databases are initialized, and the current ckan is installed into a
+python container.
+
+
+Run the tests
+=============
+
+.. parsed-literal::
+
+   ./execute.sh
+
+Or, if you wish to run a specific test, for example
+``test_get_translated`` in ``test_helpers.py``:
+
+.. parsed-literal::
+
+   docker compose exec ckan pytest --ckan-ini=test-core-circle-ci.ini ckan/tests/lib/test_helpers.py::test_get_translated
+
+
+Teardown
+========
+
+.. parsed-literal::
+
+   ./teardown.sh
+
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Install additional dependencies
+Virtual Environment based tests
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+
+Install additional dependencies
+===============================
+
 Some additional dependencies are needed to run the tests. Make sure you've
 created a config file at |ckan.ini|, then activate your
 virtual environment:
@@ -43,9 +93,9 @@ environment:
 
 .. _datastore-test-set-permissions:
 
-~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Set up the test databases
-~~~~~~~~~~~~~~~~~~~~~~~~~
+=========================
 
 Create test databases:
 
@@ -68,9 +118,9 @@ configured in ``test-core.ini`` is different from your production database.
 
 .. _solr-multi-core:
 
-~~~~~~~~~~~~~~~~~~~~~~~~~
+
 Configure Solr Multi-core
-~~~~~~~~~~~~~~~~~~~~~~~~~
+=========================
 
 The tests assume that Solr is configured 'multi-core', whereas the default
 Solr set-up is often 'single-core'. You can ask Solr for its cores status::
@@ -115,9 +165,9 @@ To enable multi-core:
        solr_url = http://127.0.0.1:8983/solr/ckan
 
 
-~~~~~~~~~~~~~
+
 Run the tests
-~~~~~~~~~~~~~
+=============
 
 To run CKAN's tests using PostgreSQL as the database, you have to give the
 ``--ckan-ini=test-core.ini`` option on the command line. This command will

test-infrastructure/README.md

@@ -0,0 +1,56 @@
+# Local testing
+
+This is a set of scripts that replicate the contents of `/.circleci/config.yml`, enabling testing on the local machine with the same setup as circleci. It requires docker with the compose plugin to be installed and on the path, as well as the `sh` shell. It should work out of the box on linux/mac, Windows is unknown.
+
+NOTE: This is not intended to be a pattern for running anything other than tests. This is not a deployment template.
+
+
+## To Run.
+
+```
+$ cd test-infrastructure
+test-infrastructure$ ./setup.sh
+[[
+* docker composee downloads images and starts the stack
+* System level dependencies (psql) are installed
+* Python dependencies are installed, ckan is installed in edit mode
+* Database creation and database initialization are done
+]]
+test-infrastructure$ ./execute.sh
+$ ./execute.sh
+2022-09-22 14:42:28,959 INFO  [ckan.cli] Using configuration file /usr/src/test-core-circle-ci.ini
+2022-09-22 14:42:28,960 INFO  [ckan.config.environment] Loading static files from public
+2022-09-22 14:42:29,371 WARNI [ckan.common] Option ckan.plugins is not declared
+2022-09-22 14:42:29,618 INFO  [ckan.config.environment] Loading templates from /usr/src/ckan/templates
+2022-09-22 14:42:30,381 INFO  [ckan.config.environment] Loading templates from /usr/src/ckan/templates
+2022-09-22 14:42:30,534 CRITI [ckan.lib.uploader] Please specify a ckan.storage_path in your config
+                        for your uploads
+2022-09-22 14:42:30,708 WARNI [ckan.config.middleware.flask_app] Extensions are excluded from CSRF protection! We allow extensions to run without CSRF protection but it will be forced future releases. Read the documentation for more information on how to add CSRF protection to your extension.
+============================================================================== test session starts ==============================================================================
+platform linux -- Python 3.7.14, pytest-7.1.3, pluggy-1.0.0 -- /usr/local/bin/python
+cachedir: .pytest_cache
+rootdir: /usr/src, configfile: pyproject.toml, testpaths: ckan, ckanext
+plugins: freezegun-0.4.2, split-0.8.0, test-groups-1.0.3, factoryboy-2.4.0, Faker-14.2.0, rerunfailures-10.2, cov-3.0.0
+collected 2883 items
+2022-09-22 14:42:45,763 INFO  [ckan.config.environment] Loading templates from /usr/src/ckan/templates
+2022-09-22 14:42:46,353 INFO  [ckan.config.environment] Loading templates from /usr/src/ckan/templates
+
+ckan/tests/test_authz.py::test_config_overrides_default PASSED                                                                                                            [  0%]
+ckan/tests/test_authz.py::test_config_override_also_works_with_prefix PASSED                                                                                              [  0%]
+ckan/tests/test_authz.py::test_unknown_permission_returns_false PASSED                                                                                                    [  0%]
+ckan/tests/test_authz.py::test_unknown_permission_not_in_config_returns_false PASSED                                                                                      [  0%]
+ckan/tests/test_authz.py::test_default_roles_`that_cascade`_to_sub_groups_is_a_list PASSED                                                                                [  0%]
+...:
+test-infrastructure$ ./teardown.sh
+```
+
+If you wish to run individual tests, it can be done with:
+```
+docker compose exec ckan pytest -vv --ckan-ini=test-core-circle-ci.ini ckan/tests/lib/test_helpers.py::test_get_translated
+```
+
+## Known issues
+
+* pytest-split causes testing errors.
+* Windows
+* Run as something other than root in the container

test-infrastructure/docker-compose.yml

@@ -0,0 +1,46 @@
+version: "3"
+networks:
+  default:
+volumes:
+  postgres-db-volume:
+
+services:
+  ckan-postgres:
+    networks:
+      - default
+    image: "postgres:10"
+    environment:
+      POSTGRES_USER: ckan
+      POSTGRES_PASSWORD: ckan
+
+  ckan-redis:
+    networks:
+      - default
+    image: "redis:3"
+
+  ckan-solr:
+    networks:
+      - default
+    image: "ckan/ckan-solr:master"
+
+  ckan:
+    image: python:3.8-bullseye
+    networks:
+      - default
+    environment:
+      HOME: /usr/src
+      CKAN_DATASTORE_POSTGRES_DB: datastore_test
+      CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+      CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+      CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+      CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+      CKAN_POSTGRES_DB: ckan_test
+      CKAN_POSTGRES_USER: ckan_default
+      CKAN_POSTGRES_PWD: pass
+      PGPASSWORD: ckan
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=~/junit/junit.xml
+    working_dir: "/usr/src"
+    volumes:
+      - "../:/usr/src"
+    command:
+      ['tail', '-f', '/etc/debian_version']

test-infrastructure/execute.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+docker compose exec ckan test-infrastructure/run_tests.sh

test-infrastructure/init_environment.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Database Creation
+psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+# Database Initialization
+ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
+psql --host=ckan-postgres --username=ckan --dbname=${CKAN_DATASTORE_POSTGRES_DB} --command="CREATE extension tablefunc;"
+ckan -c test-core-circle-ci.ini db init
+gunzip .test_durations.gz
+
+# git doesn't like having the directory owned by a different user, and
+# we're mounting this in from external, so we don't know the uid/gid.
+# this is required to build the docs
+git config --global --add safe.directory /usr/src

test-infrastructure/install_deps.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# OS Dependencies
+apt update
+apt install -y postgresql-client
+
+#Python Dependencies
+pip install -r requirements.txt
+pip install -r dev-requirements.txt
+python setup.py develop
+pip check

test-infrastructure/run_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+mkdir -p ~/junit
+python -m pytest $PYTEST_COMMON_OPTIONS
+
+# to split work across containers and run tests in parallel like circleci use commands like:
+# note that this requires multiple databases to run without data race collisions,
+#python -m pytest $PYTEST_COMMON_OPTIONS --splits 2 --group 1 --splitting-algorithm least_duration &
+#python -m pytest $PYTEST_COMMON_OPTIONS --splits 2 --group 2 --splitting-algorithm least_duration

test-infrastructure/setup.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+
+docker compose up -d
+docker compose exec ckan test-infrastructure/install_deps.sh
+docker compose exec ckan test-infrastructure/init_environment.sh

test-infrastructure/teardown.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+
+docker compose down