From 6b1c1fb7176d24ffbabbe85a951e17817c344351 Mon Sep 17 00:00:00 2001
From: Kenny Lee Sin Cheong <2530351+kleesc@users.noreply.github.com>
Date: Mon, 21 Dec 2020 10:30:59 -0500
Subject: [PATCH] Add a DEBUG flag to the executors to prevent cleanup (#631)

When set to true, DEBUG will prevent the build nodes from shutting
down after the quay-builder service is done or fails, and will prevent the
build manager from cleaning up the instances (terminating EC2
instances or deleting k8s jobs).

This will allow debugging builder node issues, and should not be set
in a production environment.

The lifetime service will still exist. i.e The instance will still
shutdown after ~2h (EC2 instances will terminate, k8s jobs will
complete)

Setting DEBUG will also affect ALLOWED_WORKER_COUNT, as the
unterminated instances/jobs will still count towards the total number
of running workers.
---
 buildman/manager/executor.py        | 18 ++++++++++++++++++
 buildman/templates/cloudconfig.json | 24 ++++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/buildman/manager/executor.py b/buildman/manager/executor.py
index 0f2d401d74..a2380d5b47 100644
--- a/buildman/manager/executor.py
+++ b/buildman/manager/executor.py
@@ -62,6 +62,21 @@ def wrapper(*args, **kwargs):
     return decorator
 
 
+def persist_for_debugging(func):
+    """
+    Wrapper for stop_builder that prevents the workers from being cleaned up (for testing purposes only)
+    """
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if self.executor_config.get("DEBUG", False):
+            logger.debug("Executor %s DEBUG set, not calling 'stop_builder()'", self.name)
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
 class ExecutorException(Exception):
     """
     Exception raised when there is a problem starting or stopping a builder.
@@ -183,6 +198,7 @@ def generate_cloud_config(
                     ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []),
                     container_runtime=self.executor_config.get("CONTAINER_RUNTIME", "docker"),
                     ca_cert=self.executor_config.get("CA_CERT", self._ca_cert()),
+                    debug=self.executor_config.get("DEBUG", False),
                 )
             )
         )
@@ -318,6 +334,7 @@ def start_builder(self, token, build_uuid):
         logger.debug("Machine with ID %s started for build %s", launched["InstanceId"], build_uuid)
         return launched["InstanceId"]
 
+    @persist_for_debugging
     def stop_builder(self, builder_id):
         try:
             ec2_conn = self._get_conn()
@@ -619,6 +636,7 @@ def start_builder(self, token, build_uuid):
         job = create_job.json()
         return job["metadata"]["name"]
 
+    @persist_for_debugging
     def stop_builder(self, builder_id):
         pods_path = "/api/v1/namespaces/%s/pods" % self.namespace
 
diff --git a/buildman/templates/cloudconfig.json b/buildman/templates/cloudconfig.json
index 717101ddc8..6b37f7c6da 100644
--- a/buildman/templates/cloudconfig.json
+++ b/buildman/templates/cloudconfig.json
@@ -111,6 +111,17 @@ WantedBy=multi-user.target
   "systemd": {
     "units": [
       {% if container_runtime == "podman" %}
+      {% if debug %}
+      {{ dockersystemd("quay-builder",
+                       worker_image,
+                       container_runtime,
+                       quay_username,
+                       quay_password,
+                       worker_tag,
+                       extra_args='--privileged --env-file /root/overrides.list -v /var/run/podman/podman.sock:/var/run/podman/podman.sock -v /etc/pki/ca-trust-source/anchors:/certs',
+                       restart_policy='no'
+                      ) | indent(6) }},
+      {% else %}
       {{ dockersystemd("quay-builder",
                        worker_image,
                        container_runtime,
@@ -121,6 +132,18 @@ WantedBy=multi-user.target
                        exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
                        restart_policy='no'
                       ) | indent(6) }},
+      {% endif %}
+      {% else %}
+      {% if debug %}
+      {{ dockersystemd("quay-builder",
+                       worker_image,
+                       container_runtime,
+                       quay_username,
+                       quay_password,
+                       worker_tag,
+                       extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /etc/pki/ca-trust-source/anchors:/certs',
+                       restart_policy='no'
+                      ) | indent(6) }},
       {% else %}
       {{ dockersystemd("quay-builder",
                        worker_image,
@@ -133,6 +156,7 @@ WantedBy=multi-user.target
                        restart_policy='no'
                       ) | indent(6) }},
       {% endif %}
+      {% endif %}
       {
         "name": "systemd-journal-gatewayd.socket",
         "enabled": true,