Add a DEBUG flag to the executors to prevent cleanup (#631) (#632)

When set to true, DEBUG will prevent the build nodes from shutting
down after the quay-builder service is done or fails, and will prevent the
build manager from cleaning up the instances (terminating EC2
instances or deleting k8s jobs).

This will allow debugging builder node issues, and should not be set
in a production environment.

The lifetime service will still exist. i.e The instance will still
shutdown after ~2h (EC2 instances will terminate, k8s jobs will
complete)

Setting DEBUG will also affect ALLOWED_WORKER_COUNT, as the
unterminated instances/jobs will still count towards the total number
of running workers.

buildman/manager/executor.py

@@ -62,6 +62,21 @@ def wrapper(*args, **kwargs):
     return decorator
 
 
+def persist_for_debugging(func):
+    """
+    Wrapper for stop_builder that prevents the workers from being cleaned up (for testing purposes only)
+    """
+
+    @wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if self.executor_config.get("DEBUG", False):
+            logger.debug("Executor %s DEBUG set, not calling 'stop_builder()'", self.name)
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
 class ExecutorException(Exception):
     """
     Exception raised when there is a problem starting or stopping a builder.
@@ -183,6 +198,7 @@ def generate_cloud_config(
                     ssh_authorized_keys=self.executor_config.get("SSH_AUTHORIZED_KEYS", []),
                     container_runtime=self.executor_config.get("CONTAINER_RUNTIME", "docker"),
                     ca_cert=self.executor_config.get("CA_CERT", self._ca_cert()),
+                    debug=self.executor_config.get("DEBUG", False),
                 )
             )
         )
@@ -318,6 +334,7 @@ def start_builder(self, token, build_uuid):
         logger.debug("Machine with ID %s started for build %s", launched["InstanceId"], build_uuid)
         return launched["InstanceId"]
 
+    @persist_for_debugging
     def stop_builder(self, builder_id):
         try:
             ec2_conn = self._get_conn()
@@ -619,6 +636,7 @@ def start_builder(self, token, build_uuid):
         job = create_job.json()
         return job["metadata"]["name"]
 
+    @persist_for_debugging
     def stop_builder(self, builder_id):
         pods_path = "/api/v1/namespaces/%s/pods" % self.namespace
 

buildman/templates/cloudconfig.json

@@ -111,6 +111,17 @@ WantedBy=multi-user.target
   "systemd": {
     "units": [
       {% if container_runtime == "podman" %}
+      {% if debug %}
+      {{ dockersystemd("quay-builder",
+                       worker_image,
+                       container_runtime,
+                       quay_username,
+                       quay_password,
+                       worker_tag,
+                       extra_args='--privileged --env-file /root/overrides.list -v /var/run/podman/podman.sock:/var/run/podman/podman.sock -v /etc/pki/ca-trust-source/anchors:/certs',
+                       restart_policy='no'
+                      ) | indent(6) }},
+      {% else %}
       {{ dockersystemd("quay-builder",
                        worker_image,
                        container_runtime,
@@ -121,6 +132,18 @@ WantedBy=multi-user.target
                        exec_stop_post=['/bin/sh -xc "/bin/sleep 120; /usr/bin/systemctl --no-block poweroff"'],
                        restart_policy='no'
                       ) | indent(6) }},
+      {% endif %}
+      {% else %}
+      {% if debug %}
+      {{ dockersystemd("quay-builder",
+                       worker_image,
+                       container_runtime,
+                       quay_username,
+                       quay_password,
+                       worker_tag,
+                       extra_args='--net=host --privileged --env-file /root/overrides.list -v /var/run/docker.sock:/var/run/docker.sock -v /etc/pki/ca-trust-source/anchors:/certs',
+                       restart_policy='no'
+                      ) | indent(6) }},
       {% else %}
       {{ dockersystemd("quay-builder",
                        worker_image,
@@ -133,6 +156,7 @@ WantedBy=multi-user.target
                        restart_policy='no'
                       ) | indent(6) }},
       {% endif %}
+      {% endif %}
       {
         "name": "systemd-journal-gatewayd.socket",
         "enabled": true,