Merge pull request #517 from okfn/517-test-db

Testing to run on own database

bin/osx-postgres-mem.sh

@@ -13,7 +13,8 @@ case $1 in
 		${PGCTL} -D ${PGDATA} init
 		${PGCTL} -D ${PGDATA} start
 		sleep 2;
-		psql -c "CREATE DATABASE ckantest;" postgres
+		psql -c "CREATE DATABASE ckan_dev;" postgres
+		psql -c "CREATE DATABASE ckan_test;" postgres
 		;;
 	stop)
 		## stop postgres

bin/travis-build

@@ -16,40 +16,43 @@ sudo apt-get update -qq
 sudo apt-get install solr-jetty postgresql-$PGVERSION
 
 # Don't require a password to access DB
-sudo sed -i -e 's/ident/trust/g' /etc/postgresql/$PGVERSION/main/pg_hba.conf 
+sudo sed -i -e 's/ident/trust/g' /etc/postgresql/$PGVERSION/main/pg_hba.conf
 
 sudo service postgresql reload
 
 pip install -r pip-requirements.txt --use-mirrors
 pip install -r pip-requirements-test.txt --use-mirrors
 
-psql -c 'CREATE DATABASE ckantest;' -U postgres
-psql -c 'CREATE DATABASE datastore;' -U postgres
+psql -c 'CREATE DATABASE ckan_test;' -U postgres
+psql -c 'CREATE DATABASE ckan_test_datastore;' -U postgres
 
 python setup.py develop
 
 # Configure CKAN's configuration file
-paster make-config ckan development.ini --no-interactive
-sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' development.ini
-sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' development.ini
-sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@\/ckantest/' development.ini
-sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@\/datastore/' development.ini
+sed -i -e 's/.*solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' test-core.ini
+sed -i -e 's/.*ckan\.site_id.*/ckan.site_id = travis_ci/' test-core.ini
+sed -i -e 's/^sqlalchemy.url.*/sqlalchemy.url = postgresql:\/\/postgres@\/ckan_test/' test-core.ini
+sed -i -e 's/.*datastore.write_url.*/ckan.datastore.write_url = postgresql:\/\/postgres@\/ckan_test_datastore/' test-core.ini
 
 # Configure Solr
 echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+# FIXME the solr schema cannot be hardcoded as it is dependent on the ckan version
 sudo cp ckan/config/solr/schema-2.0.xml /etc/solr/conf/schema.xml
 sudo service jetty restart
 
-paster --plugin=ckan db init
+paster db init -c test-core.ini
 
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
   psql -c 'CREATE USER readonlyuser;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/datastore/' development.ini
-  paster datastore set-permissions postgres
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/ckan_test_datastore/' test-core.ini
+  paster datastore set-permissions postgres -c test-core.ini
+else
+  sed -i -e 's/.*datastore.read_url.*//' test-core.ini
 fi
 
+cat test-core.ini
 
 # And finally, run the tests
 nosetests --ckan --with-pylons=test-core.ini --nologcapture ckan ckanext

ckan/config/deployment.ini_tmpl

@@ -65,7 +65,7 @@ ckan.plugins = stats json_preview recline_preview
 # Specify the database for SQLAlchemy to use:
 # * Postgres is currently required for a production CKAN deployment
 # * Sqlite (memory or file) can be used as a quick alternative for testing
-sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
+sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckan_dev
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 

doc/install-from-source.rst

@@ -100,9 +100,10 @@ prompted::
 
     sudo -u postgres createuser -S -D -R -P ckanuser
 
-Create the database (owned by ``ckanuser``), which we'll call ``ckantest``::
+Create the database (owned by ``ckanuser``), which we'll call ``ckan_dev``::
+
+    sudo -u postgres createdb -O ckanuser ckan_dev -E utf-8
 
-    sudo -u postgres createdb -O ckanuser ckantest -E utf-8
 
 4. Create a CKAN config file
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -120,7 +121,7 @@ config file::
     ``sqlalchemy.url`` line, filling in the database name, user and password you
     used::
 
-        sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckantest
+        sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckan_dev
 
     If you're using a remote host with password authentication rather than SSL
     authentication, use::

doc/test.rst

@@ -58,8 +58,19 @@ Or to run the CKAN tests and the core extensions tests together::
 Testing with PostgreSQL
 -----------------------
 
-First, make sure you have specified a PostgreSQL database with the
-``sqlalchemy.url`` parameter in your ``development.ini`` file.
+Starting in CKAN 2.1 tests are run in a separate postgres database by
+default.  You should create the test databases as follows::
+
+    sudo -u postgres createdb -O ckanuser ckan_test -E utf-8
+    sudo -u postgres createdb -O ckanuser ckan_test_datastore -E utf-8
+    # create datastore user default password `pass`
+    sudo -u postgres createuser -S -D -R -P -l readonlyuser
+    # set the permissions for readonly user
+    paster datastore set-permissions postgres -c test-core.ini
+
+
+This database connection is specified in the ``test-core.ini`` file by the
+``sqlalchemy.url`` parameter.
 
 CKAN's default nose configuration file (``test.ini``) specifies SQLite as the
 database library (it also sets ``faster_db_test_hacks``). To run the tests more
@@ -92,24 +103,12 @@ With the ``--ckan-migration`` option the tests will run using a database that
 has been created by running the migration scripts in ``ckan/migration``, which
 is how the database is created and upgraded in production.
 
-.. caution ::
-
-    Ordinarily, you should set ``development.ini`` to specify a PostgreSQL
-    database so these also get used when running ``test-core.ini``, since
-    ``test-core.ini`` inherits from ``development.ini``. If you were to change
-    the ``sqlalchemy.url`` option in your ``development.ini`` file to use
-    SQLite, the command above would actually test SQLite rather than
-    PostgreSQL, so always check the setting in ``development.ini`` to ensure
-    you are running the full tests.
-
 .. warning ::
 
    A common error when wanting to run tests against a particular database is to
    change ``sqlalchemy.url`` in ``test.ini`` or ``test-core.ini``. The problem
    is that these are versioned files and people have checked in these by
-   mistake, creating problems for other developers and the CKAN buildbot. This
-   is easily avoided by only changing ``sqlalchemy.url`` in your local
-   ``development.ini`` and testing ``--with-pylons=test-core.ini``.
+   mistake, creating problems for other developers.
 
 Common error messages
 ---------------------

test-core.ini

@@ -8,12 +8,26 @@ host = 0.0.0.0
 port = 5000
 
 [app:main]
-use = config:development.ini
+use = egg:ckan
+full_stack = true
+cache_dir = %(here)s/data
 debug = false
 
 #faster_db_test_hacks = True
+
+# Specify the database for SQLAlchemy to use:
+# * Postgres is currently required for a production CKAN deployment
+# * Sqlite (memory or file) can be used as a quick alternative for testing
+sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckan_test
 #sqlalchemy.url = sqlite:///
 
+## Datastore
+ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/ckan_test_datastore
+ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/ckan_test_datastore
+
+## Solr support
+solr_url = http://127.0.0.1:8983/solr
+
 ckan.auth.user_create_organizations = true
 ckan.auth.user_create_groups = true
 ckan.auth.create_user_via_api = false
@@ -82,6 +96,13 @@ ckan.activity_streams_email_notifications = True
 
 ckan.activity_list_limit = 15
 
+beaker.session.key = ckan
+beaker.session.secret = This_is_a_secret_or_is_it
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+
 # Logging configuration
 [loggers]
 keys = root, ckan, sqlalchemy