519 update deployment docs second try

bin/travis-build

@@ -18,10 +18,10 @@ sudo apt-get install solr-jetty postgresql-$PGVERSION
 sudo service postgresql reload
 
 # Setup postgres' users and databases
-sudo -u postgres psql -c "CREATE USER ckanuser WITH PASSWORD 'pass';"
-sudo -u postgres psql -c "CREATE USER readonlyuser WITH PASSWORD 'pass';"
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckanuser;'
-sudo -u postgres psql -c 'CREATE DATABASE ckan_test_datastore WITH OWNER ckanuser;'
+sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;'
 
 pip install -r pip-requirements.txt --use-mirrors
 pip install -r pip-requirements-test.txt --use-mirrors
@@ -39,8 +39,8 @@ paster db init -c test-core.ini
 # If Postgres >= 9.0, we don't need to use datastore's legacy mode.
 if [ $PGVERSION != '8.4' ]
 then
-  psql -c 'CREATE USER readonlyuser;' -U postgres
-  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/readonlyuser@\/ckan_test_datastore/' test-core.ini
+  psql -c 'CREATE USER datastore_default;' -U postgres
+  sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default@\/datastore_test/' test-core.ini
   paster datastore set-permissions postgres -c test-core.ini
 else
   sed -i -e 's/.*datastore.read_url.*//' test-core.ini

ckan/config/deployment.ini_tmpl

@@ -46,7 +46,14 @@ use = egg:ckan
 full_stack = true
 cache_dir = %(here)s/data
 beaker.session.key = ckan
+
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `paster make-config` generates a unique value for this each
+# time it generates a config file.
 beaker.session.secret = ${app_instance_secret}
+
+# `paster make-config` generates a unique value for this each time it generates
+# a config file.
 app_instance_uuid = ${app_instance_uuid}
 
 # List the names of CKAN extensions to activate.
@@ -65,14 +72,14 @@ ckan.plugins = stats json_preview recline_preview
 # Specify the database for SQLAlchemy to use:
 # * Postgres is currently required for a production CKAN deployment
 # * Sqlite (memory or file) can be used as a quick alternative for testing
-sqlalchemy.url = postgresql://ckanuser:pass@localhost/ckan_dev
+sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default
 #sqlalchemy.url = sqlite:///
 #sqlalchemy.url = sqlite:///%(here)s/somedb.db
 
 # Un-comment and specify the URLs for the DataStore database.
 # * Postgres is required
-#ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore
-#ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+#ckan.datastore.write_url = postgresql://ckan_default:pass@localhost/datastore_default
+#ckan.datastore.read_url = postgresql://datastore_default:pass@localhost/datastore_default
 
 # repoze.who config
 who.config_file = %(here)s/who.ini
@@ -146,7 +153,7 @@ ckan.gravatar_default = identicon
 
 ## An 'id' for the site (using, for example, when creating entries in a common search index) 
 ## If not specified derived from the site_url
-# ckan.site_id = ckan.net
+ckan.site_id = default
 
 ## API url to use (e.g. in AJAX callbacks)
 ## Enable if the API is at a different domain
@@ -241,7 +248,7 @@ ckan.feeds.author_link =
 
 # To enable local file storage:
 #ofs.impl = pairtree
-#ofs.storage_dir = /my/path/to/storage/root/directory
+#ofs.storage_dir = /var/lib/ckan/default
 
 # To enable Google cloud storage:
 #ofs.impl = google
@@ -299,24 +306,24 @@ ckan.debug_supress_header = false
 keys = root, ckan, ckanext
 
 [handlers]
-keys = console, file
+keys = console
 
 [formatters]
 keys = generic
 
 [logger_root]
 level = WARNING
-handlers = console, file
+handlers = console
 
 [logger_ckan]
 level = INFO
-handlers = console, file
+handlers = console
 qualname = ckan
 propagate = 0
 
 [logger_ckanext]
 level = DEBUG
-handlers = console, file
+handlers = console
 qualname = ckanext
 propagate = 0
 
@@ -326,11 +333,5 @@ args = (sys.stderr,)
 level = NOTSET
 formatter = generic
 
-[handler_file]
-class = logging.handlers.RotatingFileHandler
-formatter = generic
-level = NOTSET
-args = ("ckan.log", "a", 20000000, 9)
-
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

doc/conf.py

@@ -24,6 +24,33 @@
 # General configuration
 # ---------------------
 
+rst_epilog = '''
+
+.. |virtualenv_parent_dir| replace:: /usr/lib/ckan
+.. |virtualenv| replace:: |virtualenv_parent_dir|/default
+.. |activate| replace:: . |virtualenv|/bin/activate
+.. |config_parent_dir| replace:: /etc/ckan
+.. |config_dir| replace:: |config_parent_dir|/default
+.. |production.ini| replace:: |config_dir|/production.ini
+.. |development.ini| replace:: |config_dir|/development.ini
+.. |git_url| replace:: https://github.com/okfn/ckan.git
+.. |postgres| replace:: PostgreSQL
+.. |database| replace:: ckan_default
+.. |database_user| replace:: ckan_default
+.. |datastore| replace:: datastore_default
+.. |datastore_user| replace:: datastore_default
+.. |test_database| replace:: ckan_test
+.. |test_datastore| replace:: datastore_test
+.. |apache_config_file| replace:: /etc/apache2/sites-available/ckan_default
+.. |apache.wsgi| replace:: |config_dir|/apache.wsgi
+.. |data_dir| replace:: |config_dir|/data
+.. |sstore| replace:: |config_dir|/sstore
+.. |storage_parent_dir| replace:: /var/lib/ckan
+.. |storage_dir| replace:: |storage_parent_dir|/default
+.. |restart_apache| replace:: sudo service apache2 reload
+
+'''
+
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']

doc/datastore-setup.rst

@@ -25,44 +25,52 @@ List existing databases::
 
  sudo -u postgres psql -l
 
-Check that the encoding of databases is ‘UTF8’, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the datastore setup.
+Check that the encoding of databases is ``UTF8``, if not internationalisation may be a problem. Since changing the encoding of PostgreSQL may mean deleting existing databases, it is suggested that this is fixed before continuing with the datastore setup.
 
 Create users and databases
 --------------------------
 
 .. tip::
 
- As is done in the example commands below, we recommend reusing your existing
- CKAN database user (``ckanuser`` in :doc:`install-from-source`) as the
- readwrite user for your datastore database.
+ If your CKAN database and DataStore databases are on different servers, then
+ you need to create a new database user on the server where the DataStore
+ database will be created. As in :doc:`install-from-source` we'll name the
+ database user |database_user|:
 
- However, this might not be possible if the CKAN database and the DataStore
- database are on different servers. In this case, you should create a new
- database user on the server with the DataStore database::
+ .. parsed-literal::
 
-   sudo -u postgres createuser -S -D -R -P -l writeuser
+    sudo -u postgres createuser -S -D -R -P -l |database_user|
 
- Then in the commands below, replace ``ckanuser`` with ``writeuser``.
-
-Create a database user called ``readonlyuser``. This user will be given
+Create a database_user called |datastore_user|. This user will be given
 read-only access to your DataStore database in the `Set Permissions`_ step
-below::
+below:
+
+.. parsed-literal::
+
+ sudo -u postgres createuser -S -D -R -P -l |datastore_user|
 
- sudo -u postgres createuser -S -D -R -P -l readonlyuser
+Create the database (owned by |database_user|), which we'll call
+|datastore|:
 
-Create the database (owned by ``ckanuser``), which we'll call ``datastore``::
+.. parsed-literal::
 
- sudo -u postgres createdb -O ckanuser datastore -E utf-8
+ sudo -u postgres createdb -O |database_user| |datastore| -E utf-8
 
 Set URLs
 --------
 
-Now, uncomment the ``ckan.datastore.write_url`` and ``ckan.datastore.read_url`` lines in your CKAN config file and edit them if necessary::
+Now, uncomment the ``ckan.datastore.write_url`` and ``ckan.datastore.read_url``
+lines in your CKAN config file and edit them, for example:
+
+.. parsed-literal::
 
  # Datastore
  # Uncommment to set the datastore urls
- ckan.datastore.write_url = postgresql://ckanuser:pass@localhost/datastore
- ckan.datastore.read_url = postgresql://readonlyuser:pass@localhost/datastore
+ ckan.datastore.write_url = postgresql://|database_user|:pass@localhost/|datastore|
+ ckan.datastore.read_url = postgresql://|datastore_user|:pass@localhost/|datastore|
+
+Replace ``pass`` with the passwords you created for your |database_user| and
+|datastore_user| database users.
 
 Set Permissions
 ---------------
@@ -76,11 +84,13 @@ Option 1: Paster command
 
 This option is preferred if CKAN and PostgreSQL are on the same server.
 
-To set the permissions, use this paster command after you've set the database URLs (make sure to have your virtualenv activated)::
+To set the permissions, use this paster command after you've set the database URLs (make sure to have your virtualenv activated):
 
- paster datastore set-permissions postgres
+.. parsed-literal::
 
-The ``postgres`` at the end of this command should be the name of a postgres
+ paster datastore set-permissions postgres -c |development.ini|
+
+The ``postgres`` in this command should be the name of a postgres
 user with permission to create new tables and users, grant permissions, etc.
 Typically this user is called "postgres". See ``paster datastore
 set-permissions -h``.
@@ -94,15 +104,19 @@ Copy the content from the ``datastore/bin/`` directory to the database server. T
 
  python datastore_setup.py -h
 
-Once you are confident that you know the right names, set the permissions (assuming that the CKAN database is called ``ckan`` and the CKAN PostgreSQL user is called ``ckanuser``)::
+Once you are confident that you know the right names, set the permissions
+(assuming that the CKAN database is called |database| and the CKAN |postgres|
+user is called |database_user|):
+
+.. parsed-literal::
 
- python datastore_setup.py ckan datastore ckanuser ckanuser readonlyuser -p postgres
+ python datastore_setup.py |database| |datastore| |database_user| |database_user| |datastore_user| -p postgres
 
 
 Option 3: SQL script
 ~~~~~~~~~~~~~~~~~~~~
 
-This option is for more complex set-ups and requires understanding of SQL and PostgreSQL.
+This option is for more complex set-ups and requires understanding of SQL and |postgres|.
 
 Copy the ``set_permissions.sql`` file to the server that the database runs on. Make sure you set all variables in the file correctly and comment out the parts that are not needed for you set-up. Then, run the script::
 
@@ -112,22 +126,31 @@ Copy the ``set_permissions.sql`` file to the server that the database runs on. M
 3. Test the set-up
 ==================
 
-The DataStore is now set-up. To test the set-up you can list all resources that are in the DataStore::
+The DataStore is now set-up. To test the set-up, (re)start CKAN and run the
+following command to list all resources that are in the DataStore::
 
  curl -X GET "http://127.0.0.1:5000/api/3/action/datastore_search?resource_id=_table_metadata"
 
 This should return a JSON page without errors.
 
-To test the whether the set-up allows writing you can create a new resource in the DataStore. To do so, run the following command::
+To test the whether the set-up allows writing, you can create a new resource in
+the DataStore. To do so, run the following command:: 
 
  curl -X POST http://127.0.0.1:5000/api/3/action/datastore_create -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}", "fields": [ {"id": "a"}, {"id": "b"} ], "records": [ { "a": 1, "b": "xyz"}, {"a": 2, "b": "zzz"} ]}'
 
+Replace ``{YOUR-API-KEY}`` with a valid API key and ``{RESOURCE-ID}`` with a
+resource id of an existing CKAN resource.
+
 A table named after the resource id should have been created on your DataStore
-database. Visiting the following URL should return a response from the DataStore with
+database. Visiting this URL should return a response from the DataStore with
 the records inserted above::
 
  http://127.0.0.1:5000/api/3/action/datastore_search?resource_id={RESOURCE_ID}
 
+You can now delete the DataStore table with::
+
+    curl -X POST http://127.0.0.1:5000/api/3/action/datastore_delete -H "Authorization: {YOUR-API-KEY}" -d '{"resource_id": "{RESOURCE-ID}"}' 
+
 To find out more about the DataStore API, go to :doc:`datastore-api`.