From 4b56e2344c180b4c16150e1e20df30afcb4796b7 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 8 Jul 2016 15:05:27 +0200 Subject: [PATCH] [#2952] Depcreate `db load` and `db dump` Deprecates the paster commands `db load` and `db dump` in favor of PostgreSQL's `pg_dump` and `pg_restore`. `db load` and `db dump` are kept but print a warning, and their documentation is replaced with a documentation of how to dump/import the database using `pg_dump` and `pg_restore`. --- ckan/lib/cli.py | 20 +- doc/contributing/database-migrations.rst | 5 +- doc/extensions/tutorial.rst | 3 +- doc/maintaining/configuration.rst | 2 +- doc/maintaining/database-management.rst | 171 ++++++++++++++++++ doc/maintaining/getting-started.rst | 2 +- doc/maintaining/index.rst | 1 + .../installing/install-from-source.rst | 2 +- doc/maintaining/paster.rst | 111 +----------- doc/maintaining/upgrading/index.rst | 12 +- .../upgrading/upgrade-package-ckan-1-to-2.rst | 16 +- .../upgrade-package-to-minor-release.rst | 19 +- doc/maintaining/upgrading/upgrade-source.rst | 20 +- 13 files changed, 207 insertions(+), 177 deletions(-) create mode 100644 doc/maintaining/database-management.rst diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 86537863c3b..7014029670b 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -31,6 +31,18 @@ # Otherwise loggers get disabled. +def deprecation_warning(message=None): + ''' + Print a deprecation warning to STDERR. + + If ``message`` is given it is also printed to STDERR. + ''' + sys.stderr.write(u'WARNING: This function is deprecated.') + if message: + sys.stderr.write(u' ' + message.strip()) + sys.stderr.write(u'\n') + + def parse_db_config(config_key='sqlalchemy.url'): ''' Takes a config key for a database connection url and parses it into a dictionary. Expects a url like: @@ -187,10 +199,10 @@ class ManageDb(CkanCommand): search index db upgrade [version no.] - Data migrate db version - returns current version of data schema - db dump FILE_PATH - dump to a pg_dump file - db load FILE_PATH - load a pg_dump from a file + db dump FILE_PATH - dump to a pg_dump file [DEPRECATED] + db load FILE_PATH - load a pg_dump from a file [DEPRECATED] db load-only FILE_PATH - load a pg_dump from a file but don\'t do - the schema upgrade or search indexing + the schema upgrade or search indexing [DEPRECATED] db create-from-model - create database from the model (indexes not made) db migrate-filestore - migrate all uploaded data from the 2.1 filesore. ''' @@ -291,6 +303,7 @@ def _run_cmd(self, command_line): raise SystemError('Command exited with errorcode: %i' % retcode) def dump(self): + deprecation_warning(u"Use PostgreSQL's pg_dump instead.") if len(self.args) < 2: print 'Need pg_dump filepath' return @@ -300,6 +313,7 @@ def dump(self): pg_cmd = self._postgres_dump(dump_path) def load(self, only_load=False): + deprecation_warning(u"Use PostgreSQL's pg_restore instead.") if len(self.args) < 2: print 'Need pg_dump filepath' return diff --git a/doc/contributing/database-migrations.rst b/doc/contributing/database-migrations.rst index 9804cc7dbca..2cae6253cf3 100644 --- a/doc/contributing/database-migrations.rst +++ b/doc/contributing/database-migrations.rst @@ -8,9 +8,8 @@ databases to the new database schema when they upgrade their copies of CKAN. These migration scripts are kept in ``ckan.migration.versions``. When you upgrade a CKAN instance, as part of the upgrade process you run any -necessary migration scripts with the ``paster db upgrade`` command:: - - paster --plugin=ckan db upgrade --config={.ini file} +necessary migration scripts with the :ref:`paster db upgrade ` +command. A migration script should be checked into CKAN at the same time as the model changes it is related to. Before pushing the changes, ensure the tests pass diff --git a/doc/extensions/tutorial.rst b/doc/extensions/tutorial.rst index f0e4bf4ea13..2a5a88ce2e2 100644 --- a/doc/extensions/tutorial.rst +++ b/doc/extensions/tutorial.rst @@ -414,7 +414,8 @@ group named ``curators``. If you've already created a ``curators`` group and want to test what happens when the site has no ``curators`` group, you can use CKAN's command line - interface to :ref:`clean and reinitialize your database `. + interface to :ref`clean and reinitialize your database + `. Try visiting the ``/group`` page in CKAN with our ``example_iauthfunctions`` plugin activated in your CKAN config file and with no ``curators`` group in diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 67e1521f56b..5ab5824d70b 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -992,7 +992,7 @@ web interface. For example:: ckan.dumps_url = http://ckan.net/dump/ -For more information on using dumpfiles, see :ref:`paster db`. +For more information on using dumpfiles, see :ref:`datasets dump`. .. _ckan.dumps_format: diff --git a/doc/maintaining/database-management.rst b/doc/maintaining/database-management.rst new file mode 100644 index 00000000000..74cc49282c4 --- /dev/null +++ b/doc/maintaining/database-management.rst @@ -0,0 +1,171 @@ +.. _database management: + +=================== +Database Management +=================== + +.. note:: + + See :doc:`paster` for details on running the ``paster`` commands + mentioned below. + + +.. _db init: + +Initialization +-------------- + +Before you can run CKAN for the first time, you need to run ``db init`` to +initialize your database: + +.. parsed-literal:: + + paster db init -c |production.ini| + +If you forget to do this you'll see this error message in your web browser: + + 503 Service Unavailable: This site is currently off-line. Database is not + initialised. + + +.. _db clean: + +Cleaning +-------- + +.. warning:: + + This will delete all data from your CKAN database! + +You can delete everything in the CKAN database, including the tables, to start +from scratch: + +.. parsed-literal:: + + paster db clean -c |production.ini| + +After cleaning the database you must do either `initialize it`_ or `import +a previously created dump`_. + +.. _initialize it: Initialization_ +.. _import a previously created dump: `db dumping and loading`_ + + +Import and Export +----------------- + +.. _db dumping and loading: + +Dumping and Loading databases to/from a file +```````````````````````````````````````````` + +PostgreSQL offers the command line tools pg_dump_ and pg_restore_ for dumping +and restoring a database and its content to/from a file. + +For example, first dump your CKAN database:: + + sudo -u postgres pg_dump --format=custom -d ckan_default > ckan.dump + +.. warning:: + + The exported file is a complete backup of the database, and includes API + keys and other user data which may be regarded as private. So keep it + secure, like your database server. + +.. note:: + + If you've chosen a non-default database name (i.e. *not* ``ckan_default``) + then you need to adapt the commands accordingly. + +Then restore it again: + +.. parsed-literal:: + + paster db clean -c |production.ini| + sudo -u postgres pg_restore --clean --if-exists -d ckan_default < ckan.dump + +If you're importing a dump from an older version of CKAN you must :ref:`upgrade +the database schema ` after the import. + +Once the import (and a potential upgrade) is complete you should :ref:`rebuild +the search index `. + +.. note:: + + Earlier versions of CKAN offered the ``paster`` commands ``db dump`` and + ``db load``. These are still available but are deprecated in favor of + the native tools of PostgreSQL mentioned above. ``db dump`` and ``db load`` + will be removed in future versions of CKAN. + +.. _pg_dump: https://www.postgresql.org/docs/current/static/app-pgdump.html +.. _pg_restore: https://www.postgresql.org/docs/current/static/app-pgrestore.html + + +.. _datasets dump: + +Exporting Datasets to JSON Lines +```````````````````````````````` + +You can export all of your CKAN site's datasets from your database to a JSON +Lines file using ckanapi_: + +.. parsed-literal:: + + ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl + +This is useful to create a simple public listing of the datasets, with no user +information. Some simple additions to the Apache config can serve the dump +files to users in a directory listing. To do this, add these lines to your +virtual Apache config file (e.g. |apache_config_file|):: + + Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/ + + # Disable the mod_python handler for static files + + SetHandler None + Options +Indexes + + +.. warning:: + + Don't serve an SQL dump of your database (created using the ``pg_dump`` + command), as those contain private user information such as email + addresses and API keys. + +.. _ckanapi: https://github.com/ckan/ckanapi + + +.. _users dump: + +Exporting User Accounts to JSON Lines +````````````````````````````````````` + +You can export all of your CKAN site's user accounts from your database to +a JSON Lines file using ckanapi_: + +.. parsed-literal:: + + ckanapi dump users -c |production.ini| -O my_database_users.jsonl + + +.. _db upgrade: + +Upgrading +--------- + +.. warning:: + + You should :ref:`create a backup of your database ` + before upgrading it. + + To avoid problems during the database upgrade, comment out any plugins + that you have enabled in your ini file. You can uncomment them again when + the upgrade finishes. + +If you are upgrading to a new CKAN :ref:`major release ` update your +CKAN database's schema using the ``paster db upgrade`` command: + +.. parsed-literal:: + + paster db upgrade -c |production.ini| + diff --git a/doc/maintaining/getting-started.rst b/doc/maintaining/getting-started.rst index e4d05562512..38bc63509e7 100644 --- a/doc/maintaining/getting-started.rst +++ b/doc/maintaining/getting-started.rst @@ -64,7 +64,7 @@ command line with the ``create-test-data`` command: paster create-test-data -c |production.ini| If you later want to delete this test data and start again with an empty -database, you can use the ``db clean`` command, see :ref:`paster db`. +database, you can use the :ref:`db clean ` command. For a list of other command line commands for creating tests data, run:: diff --git a/doc/maintaining/index.rst b/doc/maintaining/index.rst index 57dd7db4364..01763eee6c9 100644 --- a/doc/maintaining/index.rst +++ b/doc/maintaining/index.rst @@ -11,6 +11,7 @@ installing, upgrading and configuring CKAN and its features and extensions. installing/index upgrading/index getting-started + database-management paster authorization data-viewer diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index f5f895333cf..4bcc247ca26 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -325,7 +325,7 @@ installed, we need to install and configure Solr. ------------------------- Now that you have a configuration file that has the correct settings for your -database, you can create the database tables: +database, you can :ref:`create the database tables `: .. parsed-literal:: diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst index d7490ad2da3..3ec72316443 100644 --- a/doc/maintaining/paster.rst +++ b/doc/maintaining/paster.rst @@ -254,120 +254,11 @@ Usage:: datastore set-permissions - shows a SQL script to execute -.. _paster db: - db: Manage databases ==================== -Lets you initialise, upgrade, and dump the CKAN database. - -Initialization --------------- - -Before you can run CKAN for the first time, you need to run ``db init`` to -initialize your database: - -.. parsed-literal:: - - paster db init -c |production.ini| - -If you forget to do this you'll see this error message in your web browser: - - 503 Service Unavailable: This site is currently off-line. Database is not - initialised. - -Cleaning --------- - -You can delete everything in the CKAN database, including the tables, to start -from scratch: - -.. warning:: - - This will delete all data from your CKAN database! - -.. parsed-literal:: - - paster db clean -c |production.ini| - -After cleaning the db you must do a ``db init`` or ``db load`` before CKAN will -work again. - -.. _dumping and loading: - -Dumping and Loading databases to/from a file --------------------------------------------- - -You can 'dump' (save) the exact state of the database to a file on disk and at -a later point 'load' (restore) it again. - -.. tip:: - - You can also dump the database from one CKAN instance, and then load it into - another CKAN instance on the same or another machine. This will even work if - the CKAN instance you dumped the database from is an older version of CKAN - than the one you load it into, the database will be automatically upgraded - during the load command. (But you cannot load a database from a newer - version of CKAN into an older version of CKAN.) - -To export a dump of your CKAN database: - -.. parsed-literal:: - - paster db dump -c |production.ini| my_database_dump.sql - -To load it in again, you first have to clean the database (this will delete all -data in the database!) and then load the file: - -.. parsed-literal:: - - paster db clean -c |production.ini| - paster db load -c |production.ini| my_database_dump.sql - -.. warning: - - The exported file is a complete backup of the database in plain text, and - includes API keys and other user data which may be regarded as private. So - keep it secure, like your database server. - -Exporting Datasets to JSON Lines --------------------------------- - -You can export all of your CKAN site's datasets from your database to a JSON -Lines file using the ``ckanapi dump datasets`` command: - -.. parsed-literal:: - - ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl - -This is useful to create a simple public listing of the datasets, with no user -information. Some simple additions to the Apache config can serve the dump -files to users in a directory listing. To do this, add these lines to your -virtual Apache config file (e.g. |apache_config_file|):: - - Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/ - - # Disable the mod_python handler for static files - - SetHandler None - Options +Indexes - - -.. warning:: - - Don't serve an SQL dump of your database (created using the ``paster db - dump`` command), as those contain private user information such as email - addresses and API keys. - -Exporting User Accounts to JSON Lines -------------------------------------- - -You can export all of your CKAN site's user accounts from your database to -a JSON Lines file using the ``ckanapi dump users`` command: - -.. parsed-literal:: +See :doc:`database-management`. - ckanapi dump users -c |production.ini| -O my_database_users.jsonl front-end-build: Creates and minifies css and JavaScript files ============================================================== diff --git a/doc/maintaining/upgrading/index.rst b/doc/maintaining/upgrading/index.rst index 751258614a7..d888ec2f3f5 100644 --- a/doc/maintaining/upgrading/index.rst +++ b/doc/maintaining/upgrading/index.rst @@ -109,17 +109,7 @@ to its pre-upgrade state. |activate| cd |virtualenv|/src/ckan -#. Backup your CKAN database using the ``db dump`` command, for - example: - - .. parsed-literal:: - - paster db dump --config=\ |development.ini| my_ckan_database.pg_dump - - This will create a file called ``my_ckan_database.pg_dump``, you can use the - the ``db load`` command to restore your database to the state recorded in - this file. See :ref:`paster db` for details of the ``db dump`` and ``db - load`` commands. +#. :ref:`Backup your CKAN database ` 2. Upgrade CKAN diff --git a/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst b/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst index 846a3d61998..82e514e1ead 100644 --- a/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst +++ b/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst @@ -19,9 +19,7 @@ your CKAN 1.x site, or on a different machine) and then manually migrate your database and any custom configuration, extensions or templates to your new CKAN 2.x site. We will outline the main steps for migrating below. -#. Create a dump of your CKAN 1.x database:: - - sudo -u ckanstd /var/lib/ckan/std/pyenv/bin/paster --plugin=ckan db dump db-1.x.dump --config=/etc/ckan/std/std.ini +#. :ref:`Create a dump ` of your CKAN 1.x database. #. If you want to install CKAN 2.x on the same server that your CKAN 1.x site was on, uninstall the CKAN 1.x package first:: @@ -46,15 +44,9 @@ database and any custom configuration, extensions or templates to your new CKAN |activate| cd |virtualenv|/src/ckan - Now, load your database. **This will delete any data already present in your - new CKAN 2.x database**. If you've installed CKAN 2.x on a different - machine from 1.x, first copy the database dump file to that machine. - Then run these commands: - - .. parsed-literal:: - - paster db clean -c |production.ini| - paster db load -c |production.ini| db-1.x.dump + Now :ref:`load your database dump ` into CKAN 2.x. + If you've installed CKAN 2.x on a different machine from 1.x, first copy the + database dump file to that machine. #. If you had any custom config settings in your CKAN 1.x instance that you want to copy across to your CKAN 2.x instance, then update your CKAN 2.x diff --git a/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst b/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst index 379f83f70bc..24a4a922d39 100644 --- a/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst +++ b/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst @@ -55,23 +55,8 @@ respectively. you will need to manually reinstall it. #. If there have been changes in the database schema (check the - :doc:`/changelog` to find out) you need to update your CKAN database's - schema using the ``db upgrade`` command. - - .. warning :: - - To avoid problems during the database upgrade, comment out any plugins - that you have enabled in your ini file. You can uncomment them again when - the upgrade finishes. - - For example: - - .. parsed-literal:: - - paster db upgrade --config=\ |development.ini| - - See :ref:`paster db` for details of the ``db upgrade`` - command. + :doc:`/changelog` to find out) you need to :ref:`upgrade your database + schema `. #. If there have been changes in the Solr schema (check the :doc:`/changelog` to find out) you need to restart Jetty for the changes to take effect: diff --git a/doc/maintaining/upgrading/upgrade-source.rst b/doc/maintaining/upgrading/upgrade-source.rst index b8c487f7956..4e89ff313d2 100644 --- a/doc/maintaining/upgrading/upgrade-source.rst +++ b/doc/maintaining/upgrading/upgrade-source.rst @@ -50,23 +50,9 @@ CKAN release you're upgrading to: sudo service jetty restart -#. If you are upgrading to a new :ref:`major release ` update your - CKAN database's schema using the ``db upgrade`` command. - - .. warning :: - - To avoid problems during the database upgrade, comment out any plugins - that you have enabled in your ini file. You can uncomment them again when - the upgrade finishes. - - For example: - - .. parsed-literal:: - - paster db upgrade --config=\ |development.ini| - - See :ref:`paster db` for details of the ``db upgrade`` - command. +#. If there have been changes in the database schema (check the + :doc:`/changelog` to find out) you need to :ref:`upgrade your database + schema `. #. Rebuild your search index by running the ``ckan search-index rebuild`` command: