Update cleanup ci cron (kubeflow#388)

* fix the cleanup-ci jsonnet * print http reponse * update scripts * lint
Linchin · May 9, 2019 · 62702e6 · 62702e6
1 parent daf17ed
commit 62702e6
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 8 deletions.
diff --git a/py/kubeflow/testing/cleanup_ci.py b/py/kubeflow/testing/cleanup_ci.py
@@ -263,6 +263,7 @@ def cleanup_backend_services(args):
   next_page_token = None
   expired = []
   unexpired = []
+  in_use = []
 
   while True:
     results = backends.list(project=args.project,
@@ -276,9 +277,15 @@ def cleanup_backend_services(args):
         hours=args.max_ci_deployment_resource_age_hours):
         logging.info("Deleting backend services: %s, age = %r", name, age)
         if not args.dryrun:
-          response = backends.delete(project=args.project, backendService=name)
-          logging.info("respone = %s", response)
-        expired.append(name)
+          try:
+            # An error may be thrown if the backend service is used by a urlMap.
+            response = backends.delete(project=args.project,
+                                     backendService=name).execute()
+            logging.info("response = %r", response)
+            expired.append(name)
+          except Exception as e: # pylint: disable=broad-except
+            logging.error(e)
+            in_use.append(name)
       else:
         unexpired.append(name)
 
@@ -287,7 +294,8 @@ def cleanup_backend_services(args):
     next_page_token = results["nextPageToken"]
 
   logging.info("Unexpired backend services:\n%s", "\n".join(unexpired))
-  logging.info("expired backend services:\n%s", "\n".join(expired))
+  logging.info("Deleted backend services:\n%s", "\n".join(expired))
+  logging.info("Expired but in-use backend services:\n%s", "\n".join(in_use))
 
 
 def cleanup_health_checks(args):

diff --git a/test-infra/ks_app/components/cleanup-ci-cron.jsonnet b/test-infra/ks_app/components/cleanup-ci-cron.jsonnet
@@ -7,7 +7,7 @@ local k = import "k.libsonnet";
 local cleanup = import "cleanup-ci.libsonnet";
 
 local project = "kubeflow-ci";
-local job(project) = {
+local job(project, gcBackendServices) = {
     "apiVersion": "batch/v1beta1", 
     "kind": "CronJob", 
     "metadata": {           
@@ -27,13 +27,13 @@ local job(project) = {
             app: "cleanup-ci-" + project,
           },
         },
-        spec: cleanup.jobSpec(project),
+        spec: cleanup.jobSpec(project, gcBackendServices),
       },
     }, 
 };
 
 std.prune(k.core.v1.list.new([
   // Setup 2 cron jobs for the two projects.
-  job("kubeflow-ci"),
+  job("kubeflow-ci", false),
   job("kubeflow-ci-deployment", true),
 ]))
diff --git a/test-infra/ks_app/components/cleanup-ci.libsonnet b/test-infra/ks_app/components/cleanup-ci.libsonnet
@@ -30,9 +30,9 @@
                 "-m",
                 "kubeflow.testing.cleanup_ci",
                 "--project=" + project,
+                "--gc_backend_services=" + gcBackendServices,
                 "all",
                 "--delete_script=/src/kubeflow/kubeflow/scripts/gke/delete_deployment.sh",
-                "--gc_backend_services=" + gcBackendServices,
               ],
               ]),
               "image": "gcr.io/kubeflow-ci/test-worker:v20190415-53ad3b5-dirty-5bc1cf",