From eb017b989f716739d14c3e1e968279df7372fc66 Mon Sep 17 00:00:00 2001 From: "John N. Milner" Date: Thu, 9 Jan 2020 18:20:41 -0500 Subject: [PATCH] Automatically upgrade from shared to separate dbs Depends upon https://github.com/kobotoolbox/kpi/pull/2545 --- helpers/cli.py | 3 ++ helpers/command.py | 10 ++++- helpers/upgrading.py | 89 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 helpers/upgrading.py diff --git a/helpers/cli.py b/helpers/cli.py index da4ecd9..61a3daa 100644 --- a/helpers/cli.py +++ b/helpers/cli.py @@ -98,6 +98,9 @@ def run_command(cls, command, cwd=None, polling=False): cwd=cwd) except subprocess.CalledProcessError as cpe: # Error will be display by above command. + # ^^^ this doesn't seem to be true? let's write it explicitly + # see https://docs.python.org/3/library/subprocess.html#subprocess.check_output + sys.stderr.write(cpe.output) cls.colored_print("An error has occurred", CLI.COLOR_ERROR) sys.exit(1) return stdout diff --git a/helpers/command.py b/helpers/command.py index 2b7421b..01a3054 100644 --- a/helpers/command.py +++ b/helpers/command.py @@ -10,6 +10,7 @@ from helpers.network import Network from helpers.setup import Setup from helpers.template import Template +from helpers.upgrading import migrate_single_to_two_databases class Command: @@ -40,7 +41,7 @@ def help(): " -cb, --compose-backend [docker-compose arguments]\n" " Run a docker-compose command in the back-end environment\n" " -m, --maintenance\n" - " Activate maintenance mode. All traffic is redirected to maintenance page\n" + " Activate maintenance mode. All traffic is redirected to maintenance page\n" " -v, --version\n" " Display current version\n" )) @@ -328,7 +329,7 @@ def start(cls, frontend_only=False): CLI.COLOR_ERROR) sys.exit(1) - # Make them up + # Start the back-end containers if not frontend_only: if not config_object.multi_servers or \ config_object.master_backend or config_object.slave_backend: @@ -348,6 +349,11 @@ def start(cls, frontend_only=False): CLI.run_command(backend_command, config.get("kobodocker_path")) + # If this was previously a shared-database setup, migrate to separate + # databases for KPI and KoBoCAT + migrate_single_to_two_databases() + + # Start the front-end containers if not config_object.multi_servers or config_object.frontend: frontend_command = ["docker-compose", "-f", "docker-compose.frontend.yml", diff --git a/helpers/upgrading.py b/helpers/upgrading.py new file mode 100644 index 0000000..fa18bdc --- /dev/null +++ b/helpers/upgrading.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +from __future__ import print_function, unicode_literals + +import subprocess +import sys + +from helpers.cli import CLI +from helpers.config import Config + +def migrate_single_to_two_databases(): + """ + Check the contents of the databases. If KPI's is empty or doesn't exist + while KoBoCAT's has user data, then we are migrating from a + single-database setup + """ + + config = Config().get_config() + backend_role = config.get("backend_server_role", "master") + + def _kpi_db_alias_kludge(command): + """ + Sorry, this is not very nice. See + https://github.com/kobotoolbox/kobo-docker/issues/264. + """ + set_env = 'DATABASE_URL="${KPI_DATABASE_URL}"' + return [ + "bash", "-c", + "{} {}".format(set_env, command) + ] + + kpi_run_command = ["docker-compose", + "-f", "docker-compose.frontend.yml", + "-f", "docker-compose.frontend.override.yml", + "run", "--rm", "kpi"] + if config.get("docker_prefix", "") != "": + kpi_run_command.insert(-3, "-p") + kpi_run_command.insert(-3, config.get("docker_prefix")) + + # Make sure Postgres is running + frontend_command = kpi_run_command + _kpi_db_alias_kludge(" ".join([ + "python", "manage.py", + "wait_for_database", "--retries", "30" + ])) + CLI.run_command(frontend_command, config.get("kobodocker_path")) + CLI.colored_print("The Postgres database is running!", CLI.COLOR_SUCCESS) + + frontend_command = kpi_run_command + _kpi_db_alias_kludge(" ".join([ + "python", "manage.py", + "is_database_empty", "kpi", "kobocat" + ])) + output = CLI.run_command(frontend_command, config.get("kobodocker_path")) + # TODO: read only stdout and don't consider stderr unless the exit code + # is non-zero. Currently, `output` combines both stdout and stderr + kpi_kc_db_empty = output.strip().split("\n")[-1] + + if kpi_kc_db_empty == "True\tFalse": + # KPI empty but KC is not: run the two-database upgrade script + CLI.colored_print( + "Upgrading from single-database setup to separate databases " + "for KPI and KoBoCAT", + CLI.COLOR_INFO + ) + backend_command = [ + "docker-compose", + "-f", + "docker-compose.backend.{}.yml".format(backend_role), + "-f", + "docker-compose.backend.{}.override.yml".format(backend_role), + "exec", "postgres", "bash", + "/kobo-docker-scripts/master/clone_data_from_kc_to_kpi.sh", + "--noinput" + ] + try: + subprocess.check_call( + backend_command, cwd=config.get("kobodocker_path") + ) + except subprocess.CalledProcessError: + CLI.colored_print("An error has occurred", CLI.COLOR_ERROR) + sys.exit(1) + + elif kpi_kc_db_empty not in [ + "True\tTrue", + "False\tTrue", + "False\tFalse", + ]: + # The output was invalid + CLI.colored_print("An error has occurred", CLI.COLOR_ERROR) + sys.stderr.write(kpi_kc_db_empty) + sys.exit(1)