[ckan#2952] Depcreate db load and db dump

Deprecates the paster commands `db load` and `db dump` in favor of PostgreSQL's `pg_dump` and `pg_restore`. `db load` and `db dump` are kept but print a warning, and their documentation is replaced with a documentation of how to dump/import the database using `pg_dump` and `pg_restore`.
torfsen · Jul 8, 2016 · 4b56e23 · 4b56e23
1 parent e2162f0
commit 4b56e23
Show file tree

Hide file tree

Showing 13 changed files with 207 additions and 177 deletions.
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
@@ -31,6 +31,18 @@
 #   Otherwise loggers get disabled.
 
 
+def deprecation_warning(message=None):
+    '''
+    Print a deprecation warning to STDERR.
+
+    If ``message`` is given it is also printed to STDERR.
+    '''
+    sys.stderr.write(u'WARNING: This function is deprecated.')
+    if message:
+        sys.stderr.write(u' ' + message.strip())
+    sys.stderr.write(u'\n')
+
+
 def parse_db_config(config_key='sqlalchemy.url'):
     ''' Takes a config key for a database connection url and parses it into
     a dictionary. Expects a url like:
@@ -187,10 +199,10 @@ class ManageDb(CkanCommand):
                                      search index
     db upgrade [version no.]       - Data migrate
     db version                     - returns current version of data schema
-    db dump FILE_PATH              - dump to a pg_dump file
-    db load FILE_PATH              - load a pg_dump from a file
+    db dump FILE_PATH              - dump to a pg_dump file [DEPRECATED]
+    db load FILE_PATH              - load a pg_dump from a file [DEPRECATED]
     db load-only FILE_PATH         - load a pg_dump from a file but don\'t do
-                                     the schema upgrade or search indexing
+                                     the schema upgrade or search indexing [DEPRECATED]
     db create-from-model           - create database from the model (indexes not made)
     db migrate-filestore           - migrate all uploaded data from the 2.1 filesore.
     '''
@@ -291,6 +303,7 @@ def _run_cmd(self, command_line):
             raise SystemError('Command exited with errorcode: %i' % retcode)
 
     def dump(self):
+        deprecation_warning(u"Use PostgreSQL's pg_dump instead.")
         if len(self.args) < 2:
             print 'Need pg_dump filepath'
             return
@@ -300,6 +313,7 @@ def dump(self):
         pg_cmd = self._postgres_dump(dump_path)
 
     def load(self, only_load=False):
+        deprecation_warning(u"Use PostgreSQL's pg_restore instead.")
         if len(self.args) < 2:
             print 'Need pg_dump filepath'
             return

diff --git a/doc/contributing/database-migrations.rst b/doc/contributing/database-migrations.rst
@@ -8,9 +8,8 @@ databases to the new database schema when they upgrade their copies of CKAN.
 These migration scripts are kept in ``ckan.migration.versions``.
 
 When you upgrade a CKAN instance, as part of the upgrade process you run any
-necessary migration scripts with the ``paster db upgrade`` command::
-
- paster --plugin=ckan db upgrade --config={.ini file}
+necessary migration scripts with the :ref:`paster db upgrade <db upgrade>`
+command.
 
 A migration script should be checked into CKAN at the same time as the model
 changes it is related to. Before pushing the changes, ensure the tests pass

diff --git a/doc/extensions/tutorial.rst b/doc/extensions/tutorial.rst
@@ -414,7 +414,8 @@ group named ``curators``.
 
    If you've already created a ``curators`` group and want to test what happens
    when the site has no ``curators`` group, you can use CKAN's command line
-   interface to :ref:`clean and reinitialize your database <paster db>`.
+   interface to :ref`clean and reinitialize your database
+   <database management>`.
 
 Try visiting the ``/group`` page in CKAN with our ``example_iauthfunctions``
 plugin activated in your CKAN config file and with no ``curators`` group in

diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst
@@ -992,7 +992,7 @@ web interface. For example::
 
   ckan.dumps_url = http://ckan.net/dump/
 
-For more information on using dumpfiles, see :ref:`paster db`.
+For more information on using dumpfiles, see :ref:`datasets dump`.
 
 .. _ckan.dumps_format:
 

diff --git a/doc/maintaining/database-management.rst b/doc/maintaining/database-management.rst
@@ -0,0 +1,171 @@
+.. _database management:
+
+===================
+Database Management
+===================
+
+.. note::
+
+    See :doc:`paster` for details on running the ``paster`` commands
+    mentioned below.
+
+
+.. _db init:
+
+Initialization
+--------------
+
+Before you can run CKAN for the first time, you need to run ``db init`` to
+initialize your database:
+
+.. parsed-literal::
+
+    paster db init -c |production.ini|
+
+If you forget to do this you'll see this error message in your web browser:
+
+    503 Service Unavailable:  This site is currently off-line. Database is not
+    initialised.
+
+
+.. _db clean:
+
+Cleaning
+--------
+
+.. warning::
+
+   This will delete all data from your CKAN database!
+
+You can delete everything in the CKAN database, including the tables, to start
+from scratch:
+
+.. parsed-literal::
+
+    paster db clean -c |production.ini|
+
+After cleaning the database you must do either `initialize it`_ or `import
+a previously created dump`_.
+
+.. _initialize it: Initialization_
+.. _import a previously created dump: `db dumping and loading`_
+
+
+Import and Export
+-----------------
+
+.. _db dumping and loading:
+
+Dumping and Loading databases to/from a file
+````````````````````````````````````````````
+
+PostgreSQL offers the command line tools pg_dump_ and pg_restore_ for dumping
+and restoring a database and its content to/from a file.
+
+For example, first dump your CKAN database::
+
+    sudo -u postgres pg_dump --format=custom -d ckan_default > ckan.dump
+
+.. warning::
+
+   The exported file is a complete backup of the database, and includes API
+   keys and other user data which may be regarded as private. So keep it
+   secure, like your database server.
+
+.. note::
+
+    If you've chosen a non-default database name (i.e. *not* ``ckan_default``)
+    then you need to adapt the commands accordingly.
+
+Then restore it again:
+
+.. parsed-literal::
+
+    paster db clean -c |production.ini|
+    sudo -u postgres pg_restore --clean --if-exists -d ckan_default < ckan.dump
+
+If you're importing a dump from an older version of CKAN you must :ref:`upgrade
+the database schema <db upgrade>` after the import.
+
+Once the import (and a potential upgrade) is complete you should :ref:`rebuild
+the search index <rebuild search index>`.
+
+.. note::
+
+    Earlier versions of CKAN offered the ``paster`` commands ``db dump`` and
+    ``db load``. These are still available but are deprecated in favor of
+    the native tools of PostgreSQL mentioned above. ``db dump`` and ``db load``
+    will be removed in future versions of CKAN.
+
+.. _pg_dump: https://www.postgresql.org/docs/current/static/app-pgdump.html
+.. _pg_restore: https://www.postgresql.org/docs/current/static/app-pgrestore.html
+
+
+.. _datasets dump:
+
+Exporting Datasets to JSON Lines
+````````````````````````````````
+
+You can export all of your CKAN site's datasets from your database to a JSON
+Lines file using ckanapi_:
+
+.. parsed-literal::
+
+    ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl
+
+This is useful to create a simple public listing of the datasets, with no user
+information. Some simple additions to the Apache config can serve the dump
+files to users in a directory listing. To do this, add these lines to your
+virtual Apache config file (e.g. |apache_config_file|)::
+
+    Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/
+
+    # Disable the mod_python handler for static files
+    <Location /dump>
+        SetHandler None
+        Options +Indexes
+    </Location>
+
+.. warning::
+
+   Don't serve an SQL dump of your database (created using the ``pg_dump``
+   command), as those contain private user information such as email
+   addresses and API keys.
+
+.. _ckanapi: https://github.com/ckan/ckanapi
+
+
+.. _users dump:
+
+Exporting User Accounts to JSON Lines
+`````````````````````````````````````
+
+You can export all of your CKAN site's user accounts from your database to
+a JSON Lines file using ckanapi_:
+
+.. parsed-literal::
+
+    ckanapi dump users -c |production.ini| -O my_database_users.jsonl
+
+
+.. _db upgrade:
+
+Upgrading
+---------
+
+.. warning::
+
+    You should :ref:`create a backup of your database <db dumping and loading>`
+    before upgrading it.
+
+    To avoid problems during the database upgrade, comment out any plugins
+    that you have enabled in your ini file. You can uncomment them again when
+    the upgrade finishes.
+
+If you are upgrading to a new CKAN :ref:`major release <releases>` update your
+CKAN database's schema using the ``paster db upgrade`` command:
+
+.. parsed-literal::
+
+    paster db upgrade -c |production.ini|
+
diff --git a/doc/maintaining/getting-started.rst b/doc/maintaining/getting-started.rst
@@ -64,7 +64,7 @@ command line with the ``create-test-data`` command:
    paster create-test-data -c |production.ini|
 
 If you later want to delete this test data and start again with an empty
-database, you can use the ``db clean`` command, see :ref:`paster db`.
+database, you can use the :ref:`db clean <db clean>` command.
 
 For a list of other command line commands for creating tests data, run::
 

diff --git a/doc/maintaining/index.rst b/doc/maintaining/index.rst
@@ -11,6 +11,7 @@ installing, upgrading and configuring CKAN and its features and extensions.
    installing/index
    upgrading/index
    getting-started
+   database-management
    paster
    authorization
    data-viewer

diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst
@@ -325,7 +325,7 @@ installed, we need to install and configure Solr.
 -------------------------
 
 Now that you have a configuration file that has the correct settings for your
-database, you can create the database tables:
+database, you can :ref:`create the database tables <db init>`:
 
 .. parsed-literal::
 

diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst
@@ -254,120 +254,11 @@ Usage::
     datastore set-permissions  - shows a SQL script to execute
 
 
-.. _paster db:
-
 db: Manage databases
 ====================
 
-Lets you initialise, upgrade, and dump the CKAN database.
-
-Initialization
---------------
-
-Before you can run CKAN for the first time, you need to run ``db init`` to
-initialize your database:
-
-.. parsed-literal::
-
- paster db init -c |production.ini|
-
-If you forget to do this you'll see this error message in your web browser:
-
- 503 Service Unavailable:  This site is currently off-line. Database is not
- initialised.
-
-Cleaning
---------
-
-You can delete everything in the CKAN database, including the tables, to start
-from scratch:
-
-.. warning::
-
-   This will delete all data from your CKAN database!
-
-.. parsed-literal::
-
- paster db clean -c |production.ini|
-
-After cleaning the db you must do a ``db init`` or ``db load`` before CKAN will
-work again.
-
-.. _dumping and loading:
-
-Dumping and Loading databases to/from a file
---------------------------------------------
-
-You can 'dump' (save) the exact state of the database to a file on disk and at
-a later point 'load' (restore) it again.
-
-.. tip::
-
-   You can also dump the database from one CKAN instance, and then load it into
-   another CKAN instance on the same or another machine. This will even work if
-   the CKAN instance you dumped the database from is an older version of CKAN
-   than the one you load it into, the database will be automatically upgraded
-   during the load command. (But you cannot load a database from a newer
-   version of CKAN into an older version of CKAN.)
-
-To export a dump of your CKAN database:
-
-.. parsed-literal::
-
- paster db dump -c |production.ini| my_database_dump.sql
-
-To load it in again, you first have to clean the database (this will delete all
-data in the database!) and then load the file:
-
-.. parsed-literal::
-
- paster db clean -c |production.ini|
- paster db load -c |production.ini| my_database_dump.sql
-
-.. warning:
-
-   The exported file is a complete backup of the database in plain text, and
-   includes API keys and other user data which may be regarded as private. So
-   keep it secure, like your database server.
-
-Exporting Datasets to JSON Lines
---------------------------------
-
-You can export all of your CKAN site's datasets from your database to a JSON
-Lines file using the ``ckanapi dump datasets`` command:
-
-.. parsed-literal::
-
- ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl
-
-This is useful to create a simple public listing of the datasets, with no user
-information. Some simple additions to the Apache config can serve the dump
-files to users in a directory listing. To do this, add these lines to your
-virtual Apache config file (e.g. |apache_config_file|)::
-
-    Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/
-
-    # Disable the mod_python handler for static files
-    <Location /dump>
-        SetHandler None
-        Options +Indexes
-    </Location>
-
-.. warning::
-
-   Don't serve an SQL dump of your database (created using the ``paster db
-   dump`` command), as those contain private user information such as email
-   addresses and API keys.
-
-Exporting User Accounts to JSON Lines
--------------------------------------
-
-You can export all of your CKAN site's user accounts from your database to
-a JSON Lines file using the ``ckanapi dump users`` command:
-
-.. parsed-literal::
+See :doc:`database-management`.
 
- ckanapi dump users -c |production.ini| -O my_database_users.jsonl
 
 front-end-build: Creates and minifies css and JavaScript files
 ==============================================================