From cf1c7e7c7a90239f5171a6f83608d4d40360c262 Mon Sep 17 00:00:00 2001 From: afabiani Date: Fri, 26 Feb 2016 15:04:45 +0300 Subject: [PATCH] - Backup and Restore Management Commands --- geonode/base/management/commands/backup.py | 136 ++++++++++++ geonode/base/management/commands/helpers.py | 182 ++++++++++++++++ geonode/base/management/commands/restore.py | 196 ++++++++++++++++++ geonode/base/management/commands/settings.ini | 13 ++ geonode/base/models.py | 4 +- 5 files changed, 529 insertions(+), 2 deletions(-) create mode 100644 geonode/base/management/commands/backup.py create mode 100644 geonode/base/management/commands/helpers.py create mode 100644 geonode/base/management/commands/restore.py create mode 100644 geonode/base/management/commands/settings.ini diff --git a/geonode/base/management/commands/backup.py b/geonode/base/management/commands/backup.py new file mode 100644 index 00000000000..27c81754ede --- /dev/null +++ b/geonode/base/management/commands/backup.py @@ -0,0 +1,136 @@ +######################################################################### +# +# Copyright (C) 2016 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import traceback +import os, sys +import shutil +import helpers + +from optparse import make_option + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError + +class Command(BaseCommand): + + help = 'Backup the GeoNode application data' + + option_list = BaseCommand.option_list + ( + make_option( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.'), + make_option( + '--backup-dir', + dest='backup_dir', + type="string", + help='Destination folder where to store the backup archive. It must be writable.')) + + def handle(self, **options): + ignore_errors = options.get('ignore_errors') + backup_dir = options.get('backup_dir') + + if not backup_dir or len(backup_dir) == 0: + raise CommandError("Destination folder '--backup-dir' is mandatory") + + # Create Target Folder + dir_time_suffix = helpers.get_dir_time_suffix() + target_folder = os.path.join(backup_dir, dir_time_suffix) + if not os.path.exists(target_folder): + os.makedirs(target_folder) + + # Dump Fixtures + for app_name, dump_name in zip(helpers.app_names, helpers.dump_names): + print "Dumping '"+app_name+"' into '"+dump_name+".json'." + output = open(os.path.join(target_folder,dump_name+'.json'),'w') # Point stdout at a file for dumping data to. + call_command('dumpdata',app_name,format='json',indent=2,natural=True,stdout=output) + output.close() + + # Store Media Root + media_root = settings.MEDIA_ROOT + media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT) + if not os.path.exists(media_folder): + os.makedirs(media_folder) + + helpers.copy_tree(media_root, media_folder) + print "Saved Media Files from '"+media_root+"'." + + # Store Static Root + static_root = settings.STATIC_ROOT + static_folder = os.path.join(target_folder, helpers.STATIC_ROOT) + if not os.path.exists(static_folder): + os.makedirs(static_folder) + + helpers.copy_tree(static_root, static_folder) + print "Saved Static Root from '"+static_root+"'." + + # Store Static Folders + static_folders = settings.STATICFILES_DIRS + static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS) + if not os.path.exists(static_files_folders): + os.makedirs(static_files_folders) + + for static_files_folder in static_folders: + static_folder = os.path.join(static_files_folders, os.path.basename(os.path.normpath(static_files_folder))) + if not os.path.exists(static_folder): + os.makedirs(static_folder) + + helpers.copy_tree(static_files_folder, static_folder) + print "Saved Static Files from '"+static_files_folder+"'." + + # Store Template Folders + template_folders = settings.TEMPLATE_DIRS + template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS) + if not os.path.exists(template_files_folders): + os.makedirs(template_files_folders) + + for template_files_folder in template_folders: + template_folder = os.path.join(template_files_folders, os.path.basename(os.path.normpath(template_files_folder))) + if not os.path.exists(template_folder): + os.makedirs(template_folder) + + helpers.copy_tree(template_files_folder, template_folder) + print "Saved Template Files from '"+template_files_folder+"'." + + # Store Locale Folders + locale_folders = settings.LOCALE_PATHS + locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS) + if not os.path.exists(locale_files_folders): + os.makedirs(locale_files_folders) + + for locale_files_folder in locale_folders: + locale_folder = os.path.join(locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder))) + if not os.path.exists(locale_folder): + os.makedirs(locale_folder) + + helpers.copy_tree(locale_files_folder, locale_folder) + print "Saved Locale Files from '"+locale_files_folder+"'." + + # Create Final ZIP Archive + helpers.zip_dir(target_folder, os.path.join(backup_dir, dir_time_suffix+'.zip')) + + # Cleanup Temp Folder + shutil.rmtree(target_folder) + + print "Backup Finished. Archive generated '"+os.path.join(backup_dir, dir_time_suffix+'.zip')+"'." + diff --git a/geonode/base/management/commands/helpers.py b/geonode/base/management/commands/helpers.py new file mode 100644 index 00000000000..673fe56aecd --- /dev/null +++ b/geonode/base/management/commands/helpers.py @@ -0,0 +1,182 @@ +from __future__ import with_statement +from contextlib import closing +from zipfile import ZipFile, ZIP_DEFLATED + +import traceback +import psycopg2 +import ConfigParser +import os +import time +import shutil + +MEDIA_ROOT = 'uploaded' +STATIC_ROOT = 'static_root' +STATICFILES_DIRS = 'static_dirs' +TEMPLATE_DIRS = 'template_dirs' +LOCALE_PATHS = 'locale_dirs' + +config = ConfigParser.ConfigParser() +config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'settings.ini')) + +try: + db_name = config.get('targetdb', 'dbname') + db_host = config.get('targetdb', 'host') + db_port = config.get('targetdb', 'port') + db_user = config.get('targetdb', 'user') + db_passwd = config.get('targetdb', 'passwd') +except: + pass + +app_names = config.get('fixtures', 'apps').split(',') +dump_names = config.get('fixtures', 'dumps').split(',') + +def get_db_conn(): + """Get db conn (GeoNode)""" + conn = psycopg2.connect( + "dbname='%s' user='%s' port='%s' host='%s' password='%s'" % (db_name, db_user, db_port, db_host, db_passwd) + ) + return conn + + +def patch_db(): + """Apply patch to GeoNode DB""" + conn = get_db_conn() + curs = conn.cursor() + + try: + curs.execute("ALTER TABLE base_contactrole ALTER COLUMN resource_id DROP NOT NULL") + curs.execute("ALTER TABLE base_link ALTER COLUMN resource_id DROP NOT NULL") + except Exception, err: + try: + conn.rollback() + except: + pass + + traceback.print_exc() + + conn.commit() + + +def cleanup_db(): + """Remove spurious records from GeoNode DB""" + conn = get_db_conn() + curs = conn.cursor() + + try: + curs.execute("DELETE FROM base_contactrole WHERE resource_id is NULL;") + curs.execute("DELETE FROM base_link WHERE resource_id is NULL;") + except Exception, err: + try: + conn.rollback() + except: + pass + + traceback.print_exc() + + conn.commit() + + +def load_fixture(apps, fixture_file): + from django.core import serializers + + fixture = open(fixture_file, 'rb') + + objects = serializers.deserialize('json', fixture, ignorenonexistent=True) + for obj in objects: + obj.save() + + fixture.close() + + +def get_dir_time_suffix(): + """Returns the name of a folder with the 'now' time as suffix""" + dirfmt = "%4d-%02d-%02d_%02d%02d%02d" + now = time.localtime()[0:6] + dirname = dirfmt % now + + return dirname + + +def zip_dir(basedir, archivename): + assert os.path.isdir(basedir) + with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z: + for root, dirs, files in os.walk(basedir): + #NOTE: ignore empty directories + for fn in files: + absfn = os.path.join(root, fn) + zfn = absfn[len(basedir)+len(os.sep):] #XXX: relative path + z.write(absfn, zfn) + + +def copy_tree(src, dst, symlinks=False, ignore=None): + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + + +def unzip_file(zip_file, dst): + target_folder = os.path.join(dst, os.path.splitext(os.path.basename(zip_file))[0]) + if not os.path.exists(target_folder): + os.makedirs(target_folder) + + with ZipFile(zip_file, "r") as z: + z.extractall(target_folder) + + return target_folder + + +def chmod_tree(dst, permissions=0o777): + for dirpath, dirnames, filenames in os.walk(dst): + for filename in filenames: + path = os.path.join(dirpath, filename) + os.chmod(path, permissions) + + for dirname in dirnames: + path = os.path.join(dirpath, dirname) + os.chmod(path, permissions) + + +def confirm(prompt=None, resp=False): + """prompts for yes or no response from the user. Returns True for yes and + False for no. + + 'resp' should be set to the default value assumed by the caller when + user simply types ENTER. + + >>> confirm(prompt='Create Directory?', resp=True) + Create Directory? [y]|n: + True + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: + False + >>> confirm(prompt='Create Directory?', resp=False) + Create Directory? [n]|y: y + True + + """ + + if prompt is None: + prompt = 'Confirm' + + if resp: + prompt = '%s [%s]|%s: ' % (prompt, 'y', 'n') + else: + prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y') + + while True: + ans = raw_input(prompt) + if not ans: + return resp + if ans not in ['y', 'Y', 'n', 'N']: + print 'please enter y or n.' + continue + if ans == 'y' or ans == 'Y': + return True + if ans == 'n' or ans == 'N': + return False + + diff --git a/geonode/base/management/commands/restore.py b/geonode/base/management/commands/restore.py new file mode 100644 index 00000000000..215171f0dbc --- /dev/null +++ b/geonode/base/management/commands/restore.py @@ -0,0 +1,196 @@ +######################################################################### +# +# Copyright (C) 2016 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import traceback +import os, sys +import shutil +import helpers +import tempfile + +from optparse import make_option + +from django.conf import settings +from django.core.management import call_command +from django.core.management.base import BaseCommand, CommandError + +class Command(BaseCommand): + + help = 'Restore the GeoNode application data' + + option_list = BaseCommand.option_list + ( + make_option( + '-i', + '--ignore-errors', + action='store_true', + dest='ignore_errors', + default=False, + help='Stop after any errors are encountered.'), + make_option( + '--backup-file', + dest='backup_file', + type="string", + help='Backup archive containing GeoNode data to restore.')) + + def handle(self, **options): + ignore_errors = options.get('ignore_errors') + backup_file = options.get('backup_file') + + if not backup_file or len(backup_file) == 0: + raise CommandError("Backup archive '--backup-file' is mandatory") + + if helpers.confirm(prompt='WARNING: The restore will overwrite all your GeoNode data and files. Are you sure you want to proceed?', resp=False): + # Create Target Folder + restore_folder = os.path.join(tempfile.gettempdir(), 'restore') + if not os.path.exists(restore_folder): + os.makedirs(restore_folder) + + # Extract ZIP Archive to Target Folder + target_folder = helpers.unzip_file(backup_file, restore_folder) + + # Prepare Target DB + try: + call_command('syncdb', interactive=False, load_initial_data=False) + call_command('flush', interactive=False, load_initial_data=False) + + helpers.patch_db() + except: + traceback.print_exc() + + # Restore Fixtures + for app_name, dump_name in zip(helpers.app_names, helpers.dump_names): + fixture_file = os.path.join(target_folder, dump_name+'.json') + + print "Deserializing "+fixture_file + try: + call_command('loaddata', fixture_file, app_label=app_name) + except: + #traceback.print_exc() + print "WARNING: No valid fixture data found for '"+dump_name+"'." + #helpers.load_fixture(app_name, fixture_file) + + # Restore Media Root + media_root = settings.MEDIA_ROOT + media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT) + + try: + shutil.rmtree(media_root) + except: + pass + + if not os.path.exists(media_root): + os.makedirs(media_root) + + helpers.copy_tree(media_folder, media_root) + helpers.chmod_tree(media_root) + print "Media Files Restored into '"+media_root+"'." + + # Restore Static Root + static_root = settings.STATIC_ROOT + static_folder = os.path.join(target_folder, helpers.STATIC_ROOT) + + try: + shutil.rmtree(static_root) + except: + pass + + if not os.path.exists(static_root): + os.makedirs(static_root) + + helpers.copy_tree(static_folder, static_root) + helpers.chmod_tree(static_root) + print "Static Root Restored into '"+static_root+"'." + + # Restore Static Root + static_root = settings.STATIC_ROOT + static_folder = os.path.join(target_folder, helpers.STATIC_ROOT) + + try: + shutil.rmtree(static_root) + except: + pass + + if not os.path.exists(static_root): + os.makedirs(static_root) + + helpers.copy_tree(static_folder, static_root) + helpers.chmod_tree(static_root) + print "Static Root Restored into '"+static_root+"'." + + # Restore Static Folders + static_folders = settings.STATICFILES_DIRS + static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS) + + for static_files_folder in static_folders: + + try: + shutil.rmtree(static_files_folder) + except: + pass + + if not os.path.exists(static_files_folder): + os.makedirs(static_files_folder) + + helpers.copy_tree(os.path.join(static_files_folders, os.path.basename(os.path.normpath(static_files_folder))), static_files_folder) + helpers.chmod_tree(static_files_folder) + print "Static Files Restored into '"+static_files_folder+"'." + + # Restore Template Folders + template_folders = settings.TEMPLATE_DIRS + template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS) + + for template_files_folder in template_folders: + + try: + shutil.rmtree(template_files_folder) + except: + pass + + if not os.path.exists(template_files_folder): + os.makedirs(template_files_folder) + + helpers.copy_tree(os.path.join(template_files_folders, os.path.basename(os.path.normpath(template_files_folder))), template_files_folder) + helpers.chmod_tree(template_files_folder) + print "Template Files Restored into '"+template_files_folder+"'." + + # Restore Locale Folders + locale_folders = settings.LOCALE_PATHS + locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS) + + for locale_files_folder in locale_folders: + + try: + shutil.rmtree(locale_files_folder) + except: + pass + + if not os.path.exists(locale_files_folder): + os.makedirs(locale_files_folder) + + helpers.copy_tree(os.path.join(locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder))), locale_files_folder) + helpers.chmod_tree(locale_files_folder) + print "Locale Files Restored into '"+locale_files_folder+"'." + + # Cleanup DB + try: + helpers.cleanup_db() + except: + traceback.print_exc() + + print "Restore finished. Please find restored files and dumps into: '"+target_folder+"'." + diff --git a/geonode/base/management/commands/settings.ini b/geonode/base/management/commands/settings.ini new file mode 100644 index 00000000000..6df60de7b93 --- /dev/null +++ b/geonode/base/management/commands/settings.ini @@ -0,0 +1,13 @@ +[targetdb] +dbname = geonode_rst +host = localhost +port = 5432 +user = geonode +passwd = geonode + +[fixtures] +#NOTE: Order is important +apps = people,account,avatar.avatar,base.license,base.topiccategory,base.region,base.resourcebase,base.contactrole,base.link,base.restrictioncodetype,base.spatialrepresentationtype,guardian.userobjectpermission,guardian.groupobjectpermission,layers.uploadsession,layers.layer,layers.attribute,maps.map,maps.maplayer,documents.document,taggit + +dumps = people,accounts,avatars,licenses,topiccategories,regions,resourcebases,contactroles,links,restrictioncodetypes,spatialrepresentationtypes,useropermissions,groupopermissions,uploadsessions,layers,attributes,maps,maplayers,documents,tags + diff --git a/geonode/base/models.py b/geonode/base/models.py index 77c4031f686..94427183565 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -39,7 +39,7 @@ class ContactRole(models.Model): """ ContactRole is an intermediate model to bind Profiles as Contacts to Resources and apply roles. """ - resource = models.ForeignKey('ResourceBase') + resource = models.ForeignKey('ResourceBase', blank=True, null=True) contact = models.ForeignKey(settings.AUTH_USER_MODEL) role = models.CharField(choices=ROLE_VALUES, max_length=255, help_text=_('function performed by the responsible ' 'party')) @@ -697,7 +697,7 @@ class Link(models.Model): * OGC:WFS: for WFS service links * OGC:WCS: for WCS service links """ - resource = models.ForeignKey(ResourceBase) + resource = models.ForeignKey(ResourceBase, blank=True, null=True) extension = models.CharField(max_length=255, help_text=_('For example "kml"')) link_type = models.CharField(max_length=255, choices=[(x, x) for x in LINK_TYPES]) name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"'))