- Backup and Restore Management Commands

geonode/base/management/commands/backup.py

@@ -0,0 +1,136 @@
+#########################################################################
+#
+# Copyright (C) 2016 OpenPlans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+import traceback
+import os, sys
+import shutil
+import helpers
+
+from optparse import make_option
+
+from django.conf import settings
+from django.core.management import call_command
+from django.core.management.base import BaseCommand, CommandError
+
+class Command(BaseCommand):
+
+    help = 'Backup the GeoNode application data'
+
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '-i',
+            '--ignore-errors',
+            action='store_true',
+            dest='ignore_errors',
+            default=False,
+            help='Stop after any errors are encountered.'),
+        make_option(
+            '--backup-dir',
+            dest='backup_dir',
+            type="string",
+            help='Destination folder where to store the backup archive. It must be writable.'))
+
+    def handle(self, **options):
+        ignore_errors = options.get('ignore_errors')
+        backup_dir = options.get('backup_dir')
+
+        if not backup_dir or len(backup_dir) == 0:
+            raise CommandError("Destination folder '--backup-dir' is mandatory")
+
+        # Create Target Folder
+        dir_time_suffix = helpers.get_dir_time_suffix()
+        target_folder = os.path.join(backup_dir, dir_time_suffix)
+        if not os.path.exists(target_folder):
+            os.makedirs(target_folder)
+
+        # Dump Fixtures
+        for app_name, dump_name in zip(helpers.app_names, helpers.dump_names):
+            print "Dumping '"+app_name+"' into '"+dump_name+".json'."
+            output = open(os.path.join(target_folder,dump_name+'.json'),'w') # Point stdout at a file for dumping data to.
+            call_command('dumpdata',app_name,format='json',indent=2,natural=True,stdout=output)
+            output.close()
+
+        # Store Media Root
+        media_root = settings.MEDIA_ROOT
+        media_folder = os.path.join(target_folder, helpers.MEDIA_ROOT)
+        if not os.path.exists(media_folder):
+            os.makedirs(media_folder)
+
+        helpers.copy_tree(media_root, media_folder)
+        print "Saved Media Files from '"+media_root+"'."
+
+        # Store Static Root
+        static_root = settings.STATIC_ROOT
+        static_folder = os.path.join(target_folder, helpers.STATIC_ROOT)
+        if not os.path.exists(static_folder):
+            os.makedirs(static_folder)
+
+        helpers.copy_tree(static_root, static_folder)
+        print "Saved Static Root from '"+static_root+"'."
+
+        # Store Static Folders
+        static_folders = settings.STATICFILES_DIRS
+        static_files_folders = os.path.join(target_folder, helpers.STATICFILES_DIRS)
+        if not os.path.exists(static_files_folders):
+            os.makedirs(static_files_folders)
+
+        for static_files_folder in static_folders:
+            static_folder = os.path.join(static_files_folders, os.path.basename(os.path.normpath(static_files_folder)))
+            if not os.path.exists(static_folder):
+                os.makedirs(static_folder)
+
+            helpers.copy_tree(static_files_folder, static_folder)
+            print "Saved Static Files from '"+static_files_folder+"'."
+
+        # Store Template Folders
+        template_folders = settings.TEMPLATE_DIRS
+        template_files_folders = os.path.join(target_folder, helpers.TEMPLATE_DIRS)
+        if not os.path.exists(template_files_folders):
+            os.makedirs(template_files_folders)
+
+        for template_files_folder in template_folders:
+            template_folder = os.path.join(template_files_folders, os.path.basename(os.path.normpath(template_files_folder)))
+            if not os.path.exists(template_folder):
+                os.makedirs(template_folder)
+
+            helpers.copy_tree(template_files_folder, template_folder)
+            print "Saved Template Files from '"+template_files_folder+"'."
+
+        # Store Locale Folders
+        locale_folders = settings.LOCALE_PATHS
+        locale_files_folders = os.path.join(target_folder, helpers.LOCALE_PATHS)
+        if not os.path.exists(locale_files_folders):
+            os.makedirs(locale_files_folders)
+
+        for locale_files_folder in locale_folders:
+            locale_folder = os.path.join(locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder)))
+            if not os.path.exists(locale_folder):
+                os.makedirs(locale_folder)
+
+            helpers.copy_tree(locale_files_folder, locale_folder)
+            print "Saved Locale Files from '"+locale_files_folder+"'."
+
+        # Create Final ZIP Archive
+        helpers.zip_dir(target_folder, os.path.join(backup_dir, dir_time_suffix+'.zip'))
+
+        # Cleanup Temp Folder
+        shutil.rmtree(target_folder)
+
+        print "Backup Finished. Archive generated '"+os.path.join(backup_dir, dir_time_suffix+'.zip')+"'."
+

geonode/base/management/commands/helpers.py

@@ -0,0 +1,182 @@
+from __future__ import with_statement
+from contextlib import closing
+from zipfile import ZipFile, ZIP_DEFLATED
+
+import traceback
+import psycopg2
+import ConfigParser
+import os
+import time
+import shutil
+
+MEDIA_ROOT       = 'uploaded'
+STATIC_ROOT      = 'static_root'
+STATICFILES_DIRS = 'static_dirs'
+TEMPLATE_DIRS    = 'template_dirs'
+LOCALE_PATHS     = 'locale_dirs'
+
+config = ConfigParser.ConfigParser()
+config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), 'settings.ini'))
+
+try:
+   db_name   = config.get('targetdb', 'dbname')
+   db_host   = config.get('targetdb', 'host')
+   db_port   = config.get('targetdb', 'port') 
+   db_user   = config.get('targetdb', 'user')
+   db_passwd = config.get('targetdb', 'passwd')
+except:
+   pass
+
+app_names    = config.get('fixtures', 'apps').split(',')
+dump_names   = config.get('fixtures', 'dumps').split(',')
+
+def get_db_conn():
+    """Get db conn (GeoNode)"""
+    conn = psycopg2.connect(
+        "dbname='%s' user='%s' port='%s' host='%s' password='%s'" % (db_name, db_user, db_port, db_host, db_passwd)
+    )
+    return conn
+
+
+def patch_db():
+   """Apply patch to GeoNode DB"""
+   conn   = get_db_conn()
+   curs   = conn.cursor()
+
+   try:
+      curs.execute("ALTER TABLE base_contactrole ALTER COLUMN resource_id DROP NOT NULL")
+      curs.execute("ALTER TABLE base_link ALTER COLUMN resource_id DROP NOT NULL")
+   except Exception, err:
+      try:
+         conn.rollback()
+      except:
+         pass
+
+      traceback.print_exc()
+
+   conn.commit()
+
+
+def cleanup_db():
+   """Remove spurious records from GeoNode DB"""
+   conn   = get_db_conn()
+   curs   = conn.cursor()
+
+   try:
+      curs.execute("DELETE FROM base_contactrole WHERE resource_id is NULL;")
+      curs.execute("DELETE FROM base_link WHERE resource_id is NULL;")
+   except Exception, err:
+      try:
+         conn.rollback()
+      except:
+         pass
+
+      traceback.print_exc()
+
+   conn.commit()
+
+
+def load_fixture(apps, fixture_file):
+    from django.core import serializers
+
+    fixture = open(fixture_file, 'rb')
+
+    objects = serializers.deserialize('json', fixture, ignorenonexistent=True)
+    for obj in objects:
+        obj.save()
+
+    fixture.close()
+
+
+def get_dir_time_suffix():
+   """Returns the name of a folder with the 'now' time as suffix"""
+   dirfmt = "%4d-%02d-%02d_%02d%02d%02d"
+   now = time.localtime()[0:6]
+   dirname = dirfmt % now
+
+   return dirname
+
+
+def zip_dir(basedir, archivename):
+   assert os.path.isdir(basedir)
+   with closing(ZipFile(archivename, "w", ZIP_DEFLATED)) as z:
+      for root, dirs, files in os.walk(basedir):
+         #NOTE: ignore empty directories
+         for fn in files:
+            absfn = os.path.join(root, fn)
+            zfn = absfn[len(basedir)+len(os.sep):] #XXX: relative path
+            z.write(absfn, zfn)
+
+
+def copy_tree(src, dst, symlinks=False, ignore=None):
+   for item in os.listdir(src):
+       s = os.path.join(src, item)
+       d = os.path.join(dst, item)
+       if os.path.isdir(s):
+           shutil.copytree(s, d, symlinks, ignore)
+       else:
+           shutil.copy2(s, d)
+
+
+def unzip_file(zip_file, dst):
+   target_folder = os.path.join(dst, os.path.splitext(os.path.basename(zip_file))[0])
+   if not os.path.exists(target_folder):
+      os.makedirs(target_folder)
+
+   with ZipFile(zip_file, "r") as z:
+      z.extractall(target_folder)
+
+   return target_folder
+
+
+def chmod_tree(dst, permissions=0o777):
+   for dirpath, dirnames, filenames in os.walk(dst):
+      for filename in filenames:
+         path = os.path.join(dirpath, filename)
+         os.chmod(path, permissions)
+
+      for dirname in dirnames:
+         path = os.path.join(dirpath, dirname)
+         os.chmod(path, permissions)
+
+
+def confirm(prompt=None, resp=False):
+   """prompts for yes or no response from the user. Returns True for yes and
+   False for no.
+
+   'resp' should be set to the default value assumed by the caller when
+   user simply types ENTER.
+
+   >>> confirm(prompt='Create Directory?', resp=True)
+   Create Directory? [y]|n: 
+   True
+   >>> confirm(prompt='Create Directory?', resp=False)
+   Create Directory? [n]|y: 
+   False
+   >>> confirm(prompt='Create Directory?', resp=False)
+   Create Directory? [n]|y: y
+   True
+
+   """
+
+   if prompt is None:
+       prompt = 'Confirm'
+
+   if resp:
+       prompt = '%s [%s]|%s: ' % (prompt, 'y', 'n')
+   else:
+       prompt = '%s [%s]|%s: ' % (prompt, 'n', 'y')
+
+   while True:
+       ans = raw_input(prompt)
+       if not ans:
+           return resp
+       if ans not in ['y', 'Y', 'n', 'N']:
+           print 'please enter y or n.'
+           continue
+       if ans == 'y' or ans == 'Y':
+           return True
+       if ans == 'n' or ans == 'N':
+           return False
+
+