diff --git a/.circleci-matrix.yml b/.circleci-matrix.yml deleted file mode 100644 index 8e548dc6214..00000000000 --- a/.circleci-matrix.yml +++ /dev/null @@ -1,9 +0,0 @@ -env: -- SEGMENTS="01234" -- SEGMENTS="567" -- SEGMENTS="89ab" -- SEGMENTS="cdef" - -command: -- mkdir -p $CIRCLE_TEST_REPORTS/nose -- nosetests --ckan --reset-db --with-pylons=test-core.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=$CIRCLE_TEST_REPORTS/nose/junit.xml --segments=$SEGMENTS ckan ckanext diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..552c2f47131 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,102 @@ +version: 2 +jobs: + test: + docker: + - image: python:2 + environment: + CKAN_DATASTORE_POSTGRES_DB: datastore_test + CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read + CKAN_DATASTORE_POSTGRES_READ_PWD: pass + CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write + CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass + CKAN_POSTGRES_DB: ckan_test + CKAN_POSTGRES_USER: ckan_default + CKAN_POSTGRES_PWD: pass + PGPASSWORD: ckan + NODE_TESTS_CONTAINER: 2 + NOSETEST_COMMON_OPTIONS: -v --ckan --reset-db --with-pylons=test-core-circle-ci.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=/root/junit/junit.xml ckan ckanext + - image: postgres:10 + environment: + POSTGRES_USER: ckan + POSTGRES_PASSWORD: ckan + name: ckan-postgres + - image: redis:3 + name: ckan-redis + + parallelism: 4 + + steps: + - checkout + + - run: | + # SO Dependencies + apt update + case $CIRCLE_NODE_INDEX in + $NODE_TESTS_CONTAINER) + curl -sL https://deb.nodesource.com/setup_10.x | bash - + apt install -y nodejs + npm install mocha-phantomjs@3.5.0 phantomjs@~1.9.1 + ;; + esac + apt install -y postgresql-client solr-jetty openjdk-8-jdk + + # Python Dependencies + pip install -r requirement-setuptools.txt + pip install -r requirements.txt + pip install -r dev-requirements.txt + python setup.py develop + + # SOLR config + cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml + service jetty9 restart || true # erroring out but does seem to work + + # Database Creation + psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB} + psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;" + createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB} + + # Database Initialization + paster datastore -c test-core-circle-ci.ini set-permissions | psql --host=ckan-postgres --username=ckan + paster db init -c test-core-circle-ci.ini + + # Tests Backend, split across containers by segments + - run: | + mkdir -p ~/junit + case $CIRCLE_NODE_INDEX in + 0) nosetests $NOSETEST_COMMON_OPTIONS --segments 0123 + ;; + 1) nosetests $NOSETEST_COMMON_OPTIONS --segments 4567 + ;; + 2) nosetests $NOSETEST_COMMON_OPTIONS --segments 89ab + ;; + 3) nosetests $NOSETEST_COMMON_OPTIONS --segments cdef + ;; + esac + - store_test_results: + path: ~/junit + + # Tests Frontend, only in one container + - run: + command: | + case $CIRCLE_NODE_INDEX in + $NODE_TESTS_CONTAINER) paster serve test-core-circle-ci.ini + ;; + esac + background: true + - run: + command: | + case $CIRCLE_NODE_INDEX in + $NODE_TESTS_CONTAINER) + sleep 5 + /root/project/node_modules/mocha-phantomjs/bin/mocha-phantomjs \ + http://localhost:5000/base/test/index.html + ;; + esac + +workflows: + version: 2 + build_and_test: + jobs: + - test diff --git a/circle.yml b/circle.yml deleted file mode 100644 index e560a8057a0..00000000000 --- a/circle.yml +++ /dev/null @@ -1,68 +0,0 @@ -machine: - - environment: - PIP_USE_MIRRORS: true - CKAN_POSTGRES_DB: ckan_test - CKAN_POSTGRES_USER: ckan_default - CKAN_POSTGRES_PWD: pass - CKAN_DATASTORE_POSTGRES_DB: datastore_test - CKAN_DATASTORE_POSTGRES_WRITE_USER: ckan_default - CKAN_DATASTORE_POSTGRES_READ_USER: datastore_default - CKAN_DATASTORE_POSTGRES_READ_PWD: pass - SOLR_HOME: $HOME/solr - - node: - version: 0.10.33 - -dependencies: - - pre: - - "[ -e ~/.local/bin/circleci-matrix ] - || mkdir -p ~/.local/bin - && curl -fsSL https://raw.githubusercontent.com/michaelcontento/circleci-matrix/master/src/circleci-matrix.sh -o ~/.local/bin/circleci-matrix - && chmod +x ~/.local/bin/circleci-matrix" - - override: - - pip install -r requirement-setuptools.txt - - pip install -r requirements.txt - - pip install -r dev-requirements.txt - - python setup.py develop - - post: - - "[ -e ~/nvm/v0.10.33/lib/node_modules/mocha-phantomjs ] - || npm install -g mocha-phantomjs@3.5.0 phantomjs@~1.9.1" - - cache_directories: - - ~/.local/bin/circleci-matrix - - ~/nvm/v0.10.33/lib/node_modules/mocha-phantomjs - - ~/nvm/v0.10.33/lib/node_modules/phantomjs - - ~/nvm/v0.10.33/bin/mocha-phantomjs - - ~/nvm/v0.10.33/bin/phantomjs - -database: - - post: - - sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh - - sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh - - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini - - paster datastore -c test-core.ini set-permissions | sudo -u postgres psql - - - cp -R /opt/solr-4.3.1 $SOLR_HOME - - cd $SOLR_HOME/example; java -jar start.jar >> $HOME/solr.log: - background: true - - while ! grep "Started" $HOME/solr.log; do sleep 0.1; done - - ./bin/solr_init/create_core.sh - - - paster db init -c test-core.ini - -test: - - override: - - circleci-matrix: - parallel: true - - post: - - paster serve test-core.ini: - background: true - - sleep 5 - - mocha-phantomjs http://localhost:5000/base/test/index.html diff --git a/ckan/config/environment.py b/ckan/config/environment.py index e9750d8d808..b13c290e761 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -44,7 +44,10 @@ def load_environment(global_conf, app_conf): # Required by the deliverance plugin and iATI from pylons.wsgiapp import PylonsApp import pkg_resources - find_controller_generic = PylonsApp.find_controller + find_controller_generic = getattr( + PylonsApp.find_controller, + '_old_find_controller', + PylonsApp.find_controller) # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): @@ -64,6 +67,7 @@ def find_controller(self, controller): self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) + find_controller._old_find_controller = find_controller_generic PylonsApp.find_controller = find_controller os.environ['CKAN_CONFIG'] = global_conf['__file__'] diff --git a/ckan/tests/legacy/lib/test_solr_search_index.py b/ckan/tests/legacy/lib/test_solr_search_index.py index c0a5a77bdd4..8441dfc54d4 100644 --- a/ckan/tests/legacy/lib/test_solr_search_index.py +++ b/ckan/tests/legacy/lib/test_solr_search_index.py @@ -1,5 +1,7 @@ # encoding: utf-8 +from nose.tools import assert_equal + import pysolr from ckan.common import config from ckan import model @@ -51,7 +53,6 @@ def test_0_indexing(self): def test_1_basic(self): results = self.solr.search(q='sweden', fq=self.fq) - assert len(results) == 2 - result_names = [r['name'] for r in results] - assert 'se-publications' in result_names - assert 'se-opengov' in result_names + result_names = sorted([r['name'] for r in results]) + + assert_equal([u'se-opengov', u'se-publications'], result_names) diff --git a/ckan/tests/lib/test_helpers.py b/ckan/tests/lib/test_helpers.py index e7cde117efa..ddb8600928a 100644 --- a/ckan/tests/lib/test_helpers.py +++ b/ckan/tests/lib/test_helpers.py @@ -211,7 +211,8 @@ def test_url_for_qualified_with_root_path_locale_and_script_name_env(self): @helpers.change_config('ckan.root_path', '/my/custom/path/{{LANG}}/foo') def test_url_for_with_root_path_locale_and_script_name_env(self): url = '/my/custom/path/de/foo/dataset/my_dataset' - generated_url = h.url_for('dataset.read', + generated_url = h.url_for(controller='package', + action='read', id='my_dataset', locale='de') eq_(generated_url, url) diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index ee27e0ab89f..50f57d1c025 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -285,7 +285,8 @@ def _cache_types(context): return _cache_types(context) try: - register_composite('nested', connection.connection.connection, True) + register_composite( + 'nested', connection.connection.connection, True) except AttributeError: register_composite('nested', connection.connection, True) diff --git a/ckanext/imageview/tests/test_view.py b/ckanext/imageview/tests/test_view.py index 5bd3bc977c3..15f6af64224 100644 --- a/ckanext/imageview/tests/test_view.py +++ b/ckanext/imageview/tests/test_view.py @@ -9,22 +9,7 @@ class TestImageView(helpers.FunctionalTestBase): - - @classmethod - def setup_class(cls): - - super(TestImageView, cls).setup_class() - - if not p.plugin_loaded('image_view'): - p.load('image_view') - - @classmethod - def teardown_class(cls): - p.unload('image_view') - - super(TestImageView, cls).teardown_class() - - helpers.reset_db() + _load_plugins = ['image_view'] @helpers.change_config('ckan.views.default_views', '') def test_view_shown_on_resource_page_with_image_url(self): diff --git a/test-core-circle-ci.ini b/test-core-circle-ci.ini new file mode 100644 index 00000000000..a558853e415 --- /dev/null +++ b/test-core-circle-ci.ini @@ -0,0 +1,49 @@ +[server:main] +use = config:test-core.ini + +[app:main] +use = config:test-core.ini + +ckan.datastore.write_url = postgresql://datastore_write:pass@ckan-postgres/datastore_test +ckan.datastore.read_url = postgresql://datastore_read:pass@ckan-postgres/datastore_test + +ckan.redis.url = redis://ckan-redis:6379/1 + +sqlalchemy.url = postgresql://ckan_default:pass@ckan-postgres/ckan_test + +solr_url = http://localhost:8080/solr + +[loggers] +keys = root, ckan, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_ckan] +qualname = ckan +handlers = +level = INFO + +[logger_sqlalchemy] +handlers = +qualname = sqlalchemy.engine +level = WARNING +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARNING" logs neither. + +[handler_console] +class = StreamHandler +args = (sys.stdout,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s