Merge pull request #8468 from rtibbles/preseed_all_the_dbs

Preseed all the databases

Makefile

@@ -164,13 +164,7 @@ writeversion:
 	@echo "Current version is now `cat kolibri/VERSION`"
 
 preseeddb:
-	$(eval TEMPKHOME := $(shell mktemp -d /tmp/kolibri-preseeddb-XXXXXXXX))
-	PYTHONPATH=".:$$PYTHONPATH" KOLIBRI_HOME=$(TEMPKHOME) python -m kolibri manage migrate
-	yes yes | PYTHONPATH=".:$$PYTHONPATH" KOLIBRI_HOME=$(TEMPKHOME) python -m kolibri manage deprovision
-	mkdir kolibri/dist/home
-	mv $(TEMPKHOME)/db.sqlite3 kolibri/dist/home/db.sqlite3
-	mv $(TEMPKHOME)/notifications.sqlite3 kolibri/dist/home/notifications.sqlite3
-	rm -r $(TEMPKHOME)
+	PYTHONPATH=".:$PYTHONPATH" python build_tools/preseed_home.py
 
 setrequirements:
 	rm -r requirements.txt || true # remove requirements.txt
@@ -182,7 +176,7 @@ buildconfig:
 	git checkout -- kolibri/utils/build_config # restore __init__.py
 	python build_tools/customize_build.py
 
-dist: setrequirements writeversion staticdeps staticdeps-cext buildconfig i18n-extract-frontend assets i18n-django-compilemessages
+dist: setrequirements writeversion staticdeps staticdeps-cext buildconfig i18n-extract-frontend assets i18n-django-compilemessages preseeddb
 	python setup.py sdist --format=gztar > /dev/null # silence the sdist output! Too noisy!
 	python setup.py bdist_wheel
 	ls -l dist

build_tools/preseed_home.py

@@ -0,0 +1,32 @@
+import os
+import shutil
+import tempfile
+
+temphome = tempfile.mkdtemp()
+os.environ["KOLIBRI_HOME"] = temphome
+
+from kolibri.main import initialize  # noqa E402
+from kolibri.deployment.default.sqlite_db_names import (  # noqa E402
+    ADDITIONAL_SQLITE_DATABASES,
+)
+from django.core.management import call_command  # noqa E402
+
+move_to = os.path.join(os.path.dirname(__file__), "..", "kolibri", "dist", "home")
+shutil.rmtree(move_to, ignore_errors=True)
+os.mkdir(move_to)
+
+print("Generating preseeded home data in {}".format(temphome))
+
+initialize()
+call_command(
+    "deprovision", "--destroy-all-user-data", "--permanent-irrevocable-data-loss"
+)
+
+shutil.move(os.path.join(temphome, "db.sqlite3"), move_to)
+
+for db_name in ADDITIONAL_SQLITE_DATABASES:
+    shutil.move(os.path.join(temphome, "{}.sqlite3".format(db_name)), move_to)
+
+print("Moved all preseeded home data to {}".format(move_to))
+
+shutil.rmtree(temphome)

kolibri/core/auth/management/commands/deprovision.py

@@ -40,6 +40,20 @@
 class Command(AsyncCommand):
     help = "Delete all facility user data from the local database, and put it back to a clean state (but leaving content as-is)."
 
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--destroy-all-user-data",
+            action="store_true",
+            dest="confirmation1",
+            default=False,
+        )
+        parser.add_argument(
+            "--permanent-irrevocable-data-loss",
+            action="store_true",
+            dest="confirmation2",
+            default=False,
+        )
+
     def deprovision(self):
         with DisablePostDeleteSignal(), self.start_progress(
             total=len(MODELS_TO_DELETE)
@@ -60,13 +74,17 @@ def handle_async(self, *args, **options):
         except server.NotRunning:
             pass
 
-        # ensure the user REALLY wants to do this!
-        confirm_or_exit(
-            "Are you sure you wish to deprovision your database? This will DELETE ALL USER DATA!"
-        )
-        confirm_or_exit(
-            "ARE YOU SURE? If you do this, there is no way to recover the user data on this device."
-        )
+        if not options["confirmation1"]:
+            # ensure the user REALLY wants to do this!
+            confirm_or_exit(
+                "Are you sure you wish to deprovision your database? This will DELETE ALL USER DATA!"
+            )
+
+        if not options["confirmation2"]:
+            # ensure the user REALLY REALLY wants to do this!
+            confirm_or_exit(
+                "ARE YOU SURE? If you do this, there is no way to recover the user data on this device."
+            )
 
         print("Proceeding with deprovisioning. Deleting all user data.")
         self.deprovision()

kolibri/utils/main.py

@@ -23,6 +23,7 @@
 from kolibri.core.tasks.main import import_tasks_module_from_django_apps
 from kolibri.core.upgrade import matches_version
 from kolibri.core.upgrade import run_upgrades
+from kolibri.deployment.default.sqlite_db_names import ADDITIONAL_SQLITE_DATABASES
 from kolibri.plugins.utils import autoremove_unavailable_plugins
 from kolibri.plugins.utils import check_plugin_config_file_location
 from kolibri.plugins.utils import enable_new_default_plugins
@@ -160,6 +161,29 @@ def _setup_django():
         raise
 
 
+def _copy_preseeded_db(db_name, target=None):
+    target = target or "{}.sqlite3".format(db_name)
+    target = os.path.join(KOLIBRI_HOME, target)
+    if not os.path.exists(target):
+        try:
+            import kolibri.dist
+
+            db_path = os.path.join(
+                os.path.dirname(kolibri.dist.__file__),
+                "home/{}.sqlite3".format(db_name),
+            )
+            shutil.copy(db_path, target)
+            logger.info(
+                "Copied preseeded database from {} to {}".format(db_path, target)
+            )
+        except (ImportError, IOError, OSError):
+            logger.warning(
+                "Unable to copy pre-migrated database from {} to {}".format(
+                    db_path, target
+                )
+            )
+
+
 def _upgrades_before_django_setup(updated, version):
     if version and updated:
         check_plugin_config_file_location(version)
@@ -170,35 +194,17 @@ def _upgrades_before_django_setup(updated, version):
         # dbbackup relies on settings.INSTALLED_APPS
         enable_new_default_plugins()
 
-    if not version and OPTIONS["Database"]["DATABASE_ENGINE"] == "sqlite":
-        # If there is no registered version, and we are using sqlite,
+    if OPTIONS["Database"]["DATABASE_ENGINE"] == "sqlite":
+        # If we are using sqlite,
         # we can shortcut migrations by using the preseeded databases
         # that we bundle in the Kolibri whl file.
-        logger.info("Attempting to setup using pre-migrated databases")
-        try:
-            import kolibri.dist
+        if not version:
+            logger.info("Attempting to setup using pre-migrated databases")
 
-            main_db_path = os.path.join(kolibri.dist.__file__, "home/db.sqlite3")
-            shutil.copy(main_db_path, KOLIBRI_HOME)
-        except (ImportError, IOError, OSError):
-            logger.warning(
-                "Unable to copy pre-migrated database from {} to {}".format(
-                    main_db_path, KOLIBRI_HOME
-                )
-            )
-        try:
-            import kolibri.dist
+        _copy_preseeded_db("db", target=OPTIONS["Database"]["DATABASE_NAME"])
 
-            notifications_db_path = os.path.join(
-                kolibri.dist.__file__, "home/notifications.sqlite3"
-            )
-            shutil.copy(notifications_db_path, KOLIBRI_HOME)
-        except (ImportError, IOError, OSError):
-            logger.warning(
-                "Unable to copy pre-migrated database from {} to {}".format(
-                    notifications_db_path, KOLIBRI_HOME
-                )
-            )
+        for db_name in ADDITIONAL_SQLITE_DATABASES:
+            _copy_preseeded_db(db_name)
 
 
 def _upgrades_after_django_setup(updated, version):