Merge pull request #4430 from wardi/circle-v2

Circle v2 w/parallel containers

.circleci/config.yml

@@ -0,0 +1,102 @@
+version: 2
+jobs:
+  test:
+    docker:
+      - image: python:2
+        environment:
+          CKAN_DATASTORE_POSTGRES_DB: datastore_test
+          CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+          CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+          CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+          CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+          CKAN_POSTGRES_DB: ckan_test
+          CKAN_POSTGRES_USER: ckan_default
+          CKAN_POSTGRES_PWD: pass
+          PGPASSWORD: ckan
+          NODE_TESTS_CONTAINER: 2
+          NOSETEST_COMMON_OPTIONS: -v --ckan --reset-db --with-pylons=test-core-circle-ci.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=/root/junit/junit.xml ckan ckanext
+      - image: postgres:10
+        environment:
+          POSTGRES_USER: ckan
+          POSTGRES_PASSWORD: ckan
+        name: ckan-postgres
+      - image: redis:3
+        name: ckan-redis
+
+    parallelism: 4
+
+    steps:
+      - checkout
+
+      - run: |
+          # SO Dependencies
+          apt update
+          case $CIRCLE_NODE_INDEX in
+            $NODE_TESTS_CONTAINER)
+            curl -sL https://deb.nodesource.com/setup_10.x | bash -
+            apt install -y nodejs
+            npm install mocha-phantomjs@3.5.0 phantomjs@~1.9.1
+            ;;
+          esac
+          apt install -y postgresql-client solr-jetty openjdk-8-jdk
+
+          # Python Dependencies
+          pip install -r requirement-setuptools.txt
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          python setup.py develop
+
+          # SOLR config
+          cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
+          service jetty9 restart || true  # erroring out but does seem to work
+
+          # Database Creation
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+          # Database Initialization
+          paster datastore -c test-core-circle-ci.ini set-permissions | psql --host=ckan-postgres --username=ckan
+          paster db init -c test-core-circle-ci.ini
+
+      # Tests Backend, split across containers by segments
+      - run: |
+          mkdir -p ~/junit
+          case $CIRCLE_NODE_INDEX in
+            0) nosetests $NOSETEST_COMMON_OPTIONS --segments 0123
+            ;;
+            1) nosetests $NOSETEST_COMMON_OPTIONS --segments 4567
+            ;;
+            2) nosetests $NOSETEST_COMMON_OPTIONS --segments 89ab
+            ;;
+            3) nosetests $NOSETEST_COMMON_OPTIONS --segments cdef
+            ;;
+          esac
+      - store_test_results:
+          path: ~/junit
+
+      # Tests Frontend, only in one container
+      - run:
+          command: |
+            case $CIRCLE_NODE_INDEX in
+              $NODE_TESTS_CONTAINER) paster serve test-core-circle-ci.ini
+              ;;
+            esac
+          background: true
+      - run:
+          command: |
+            case $CIRCLE_NODE_INDEX in
+              $NODE_TESTS_CONTAINER)
+                sleep 5
+                /root/project/node_modules/mocha-phantomjs/bin/mocha-phantomjs \
+                  http://localhost:5000/base/test/index.html
+              ;;
+            esac
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - test

ckan/config/environment.py

@@ -45,7 +45,10 @@ def load_environment(global_conf, app_conf):
     # Required by the deliverance plugin and iATI
     from pylons.wsgiapp import PylonsApp
     import pkg_resources
-    find_controller_generic = PylonsApp.find_controller
+    find_controller_generic = getattr(
+        PylonsApp.find_controller,
+        '_old_find_controller',
+        PylonsApp.find_controller)
 
     # This is from pylons 1.0 source, will monkey-patch into 0.9.7
     def find_controller(self, controller):
@@ -65,6 +68,7 @@ def find_controller(self, controller):
             self.controller_classes[controller] = mycontroller
             return mycontroller
         return find_controller_generic(self, controller)
+    find_controller._old_find_controller = find_controller_generic
     PylonsApp.find_controller = find_controller
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']

ckan/tests/legacy/lib/test_solr_search_index.py

@@ -1,5 +1,7 @@
 # encoding: utf-8
 
+from nose.tools import assert_equal
+
 import pysolr
 from ckan.common import config
 from ckan import model
@@ -51,7 +53,6 @@ def test_0_indexing(self):
 
     def test_1_basic(self):
         results = self.solr.search(q='sweden', fq=self.fq)
-        assert len(results) == 2
-        result_names = [r['name'] for r in results]
-        assert 'se-publications' in result_names
-        assert 'se-opengov' in result_names
+        result_names = sorted([r['name'] for r in results])
+
+        assert_equal([u'se-opengov', u'se-publications'], result_names)

ckanext/imageview/tests/test_view.py

@@ -9,22 +9,7 @@
 
 
 class TestImageView(helpers.FunctionalTestBase):
-
-    @classmethod
-    def setup_class(cls):
-
-        super(TestImageView, cls).setup_class()
-
-        if not p.plugin_loaded('image_view'):
-            p.load('image_view')
-
-    @classmethod
-    def teardown_class(cls):
-        p.unload('image_view')
-
-        super(TestImageView, cls).teardown_class()
-
-        helpers.reset_db()
+    _load_plugins = ['image_view']
 
     @helpers.change_config('ckan.views.default_views', '')
     def test_view_shown_on_resource_page_with_image_url(self):

test-core-circle-ci.ini

@@ -0,0 +1,49 @@
+[server:main]
+use = config:test-core.ini
+
+[app:main]
+use = config:test-core.ini
+
+ckan.datastore.write_url = postgresql://datastore_write:pass@ckan-postgres/datastore_test
+ckan.datastore.read_url = postgresql://datastore_read:pass@ckan-postgres/datastore_test
+
+ckan.redis.url = redis://ckan-redis:6379/1
+
+sqlalchemy.url = postgresql://ckan_default:pass@ckan-postgres/ckan_test
+
+solr_url = http://localhost:8080/solr
+
+[loggers]
+keys = root, ckan, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_ckan]
+qualname = ckan
+handlers =
+level = INFO
+
+[logger_sqlalchemy]
+handlers =
+qualname = sqlalchemy.engine
+level = WARNING
+# "level = INFO" logs SQL queries.
+# "level = DEBUG" logs SQL queries and results.
+# "level = WARNING" logs neither.
+
+[handler_console]
+class = StreamHandler
+args = (sys.stdout,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s