From c421ba21f4a0976a354bbdb12fc839fec415b57b Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 16:31:27 +0000 Subject: [PATCH 1/6] add deploy.db-restore task Co-authored-by: Scott Morningstar Co-authored-by: Jason Judkins --- kubesae/ansible/deploy.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/kubesae/ansible/deploy.py b/kubesae/ansible/deploy.py index 278bc15..fd149ab 100644 --- a/kubesae/ansible/deploy.py +++ b/kubesae/ansible/deploy.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import invoke @@ -107,13 +108,39 @@ def ansible_playbook(c, name, extra="", verbosity=1, limit=""): limit = f"-l{c.config.env}" v_flag = get_verbosity_flag(verbosity) with c.cd("deploy/"): - c.run( - f"ansible-playbook {name} {limit} {extra} {v_flag}", - env=shell_env + + +@invoke.task(pre=[install_requirements]) +def ansible_db_restore(c, filename, name="", extra="", verbosity=0, limit=""): + """Restore PostgreSQL database with an Ansible db-restore.yaml playbook. + + Params: + filename: The custom-formatted PostgreSQL database archive path + name: The name of the Ansible playbook to run, including the extension + extra: Additional command line arguments to ansible-playbook + verbosity: integer level of verbosity from 0 to 4 (most verbose) + limit: The limit passed to underlying ansible-playbook + + Usage: inv deploy.db-restore --filename=mydbarchive.pgdump + + """ + if not name: + name = ( + "db-restore.yaml" + if os.path.exists("deploy/db-restore.yaml") + else "db-restore.yml" ) + archive_path = Path(filename) + if not archive_path.exists(): + archive_path = archive_path / "deploy" + extra = [extra, f"-e k8s_restore_local_archive_path={archive_path.resolve()}"] + deploy["playbook"]( + c, name=name, extra=" ".join(extra), verbosity=verbosity, limit=limit + ) deploy = invoke.Collection("deploy") deploy.add_task(install_requirements, "install") deploy.add_task(ansible_deploy, "deploy") deploy.add_task(ansible_playbook, "playbook") +deploy.add_task(ansible_db_restore, "db-restore") From 2500585c0e9331654468fe13939341333b5f29d0 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 16:35:52 +0000 Subject: [PATCH 2/6] restore playbook run --- kubesae/ansible/deploy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kubesae/ansible/deploy.py b/kubesae/ansible/deploy.py index fd149ab..550b76a 100644 --- a/kubesae/ansible/deploy.py +++ b/kubesae/ansible/deploy.py @@ -108,6 +108,7 @@ def ansible_playbook(c, name, extra="", verbosity=1, limit=""): limit = f"-l{c.config.env}" v_flag = get_verbosity_flag(verbosity) with c.cd("deploy/"): + c.run(f"ansible-playbook {name} {limit} {extra} {v_flag}", env=shell_env) @invoke.task(pre=[install_requirements]) From aa953dbcc17ba4a58f42b157c62325ad49a2f120 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 16:41:38 +0000 Subject: [PATCH 3/6] add release notes --- RELEASES.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASES.rst b/RELEASES.rst index 75cc9e0..2341c8e 100644 --- a/RELEASES.rst +++ b/RELEASES.rst @@ -1,6 +1,10 @@ Releases ======== +TBD +~~~~~~~~~~~~~~~~~~~ +* Add simpler helper to restore backups (``deploy.db-restore``) when testing disaster recovery tasks. + v0.0.21, 2022-29-08 ~~~~~~~~~~~~~~~~~~~ * Adds a utility `util.scale-app` which assists with scaling a namespace's app and celery deployments and celery-beat statefulset. From eaa6968062f05f6706c27a55f7d00e59cac2d33b Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 18:03:56 +0000 Subject: [PATCH 4/6] don't try to fix path --- kubesae/ansible/deploy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/kubesae/ansible/deploy.py b/kubesae/ansible/deploy.py index 550b76a..768539f 100644 --- a/kubesae/ansible/deploy.py +++ b/kubesae/ansible/deploy.py @@ -132,8 +132,6 @@ def ansible_db_restore(c, filename, name="", extra="", verbosity=0, limit=""): else "db-restore.yml" ) archive_path = Path(filename) - if not archive_path.exists(): - archive_path = archive_path / "deploy" extra = [extra, f"-e k8s_restore_local_archive_path={archive_path.resolve()}"] deploy["playbook"]( c, name=name, extra=" ".join(extra), verbosity=verbosity, limit=limit From 580f60c5c22c7e19c9c74c2ba3e7201fceeb23e1 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 19:25:55 +0000 Subject: [PATCH 5/6] add optional support for a destination filename --- kubesae/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/kubesae/utils.py b/kubesae/utils.py index e5b1ab1..6e91a13 100644 --- a/kubesae/utils.py +++ b/kubesae/utils.py @@ -26,7 +26,7 @@ def result_to_json(result: invoke.Result): @invoke.task(name="get_db_backup") def get_backup_from_hosting( - c, latest="daily", profile="caktus", backup_name=None, list=False + c, latest="daily", profile="caktus", backup_name=None, list=False, dest="" ): """Downloads a backup from the caktus hosting services bucket @@ -37,6 +37,7 @@ def get_backup_from_hosting( profile (str, optional): The AWS profile to allow access to the s3 bucket. DEFAULT: "caktus" backup_name(str, optional): A specific backup filename. list(bool, optional): If set, will list the contents of the bucket for the projects folder and exit. + dest (str, optional): Output filename Usage: $ inv utils.get-db-backup @@ -55,6 +56,8 @@ def get_backup_from_hosting( Will list all of the backup files using the a locally configured AWS_PROFILE named "client-aws" """ + if c.config.get("hosting_services_backup_profile"): + profile = c.config.hosting_services_backup_profile if c.config.get("hosting_services_backup_bucket"): bucket = f"s3://{c.config.hosting_services_backup_bucket.strip('/')}" else: @@ -95,12 +98,12 @@ def get_backup_from_hosting( f"{latest}-{c.config.hosting_services_backup_folder}-{dates[-1]}.pgdump" ) + if not dest: + dest = backup_name if not backup_name: print(f"No backup matching a latest of {latest} could be found.") return - c.run( - f"aws s3 cp {bucket_folder}/{backup_name} ./{backup_name} --profile {profile}" - ) + c.run(f"aws s3 cp {bucket_folder}/{backup_name} ./{dest} --profile {profile}") @invoke.task From aa936c5b6f9aefb3a36b38471daed0b825c0846e Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Thu, 1 Dec 2022 20:16:11 +0000 Subject: [PATCH 6/6] add v0.1.0 release notes --- RELEASES.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/RELEASES.rst b/RELEASES.rst index 2341c8e..8780e55 100644 --- a/RELEASES.rst +++ b/RELEASES.rst @@ -1,9 +1,11 @@ Releases ======== -TBD +v0.1.0, 2022-12-01 ~~~~~~~~~~~~~~~~~~~ -* Add simpler helper to restore backups (``deploy.db-restore``) when testing disaster recovery tasks. +* Add simpler helper to restore backups (``deploy.db-restore``) when testing disaster recovery tasks (#49) +* Fix ``utils.get-db-dump`` breaking when output includes kubectl warnings (#48) +* Use pre-commit for: black, flake8, isort, prettier (#50) v0.0.21, 2022-29-08 ~~~~~~~~~~~~~~~~~~~