Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(k8s): handle status of deleted Pod correctly #1084

Merged
merged 1 commit into from
May 6, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
25 changes: 16 additions & 9 deletions enterprise_gateway/services/processproxies/k8s.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,26 +111,33 @@ def terminate_container_resources(self):

# Delete the namespace or pod...
try:
# What gets returned from this call is a 'V1Status'. It looks a bit like JSON but appears to be
# intentionally obfuscated. Attempts to load the status field fail due to malformed json. As a
# result, we'll see if the status field contains either 'Succeeded' or 'Failed' - since that should
# indicate the phase value.
status = None
termination_stati = ["Succeeded", "Failed", "Terminating"]

if self.delete_kernel_namespace and not self.kernel_manager.restarting:
# Status is a return value for calls that don't return other objects.
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#status-v1-meta
v1_status = client.CoreV1Api().delete_namespace(
name=self.kernel_namespace, body=body
)
if v1_status:
status = v1_status.status
else:
v1_status = client.CoreV1Api().delete_namespaced_pod(
# Deleting a Pod will return a v1.Pod if found and its status will be a PodStatus containing
# a phase string property
# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#podstatus-v1-core
v1_pod = client.CoreV1Api().delete_namespaced_pod(
namespace=self.kernel_namespace, body=body, name=self.container_name
)
if v1_status and v1_status.status:
termination_stati = ["Succeeded", "Failed", "Terminating"]
if any(status in v1_status.status for status in termination_stati):
if v1_pod and v1_pod.status:
status = v1_pod.status.phase

if status:
if any(s in status for s in termination_stati):
result = True

if not result:
self.log.warning(f"Unable to delete {object_name}: {v1_status}")
self.log.warning(f"Unable to delete {object_name}: {status}")
except Exception as err:
if isinstance(err, client.rest.ApiException) and err.status == 404:
result = True # okay if its not found
Expand Down