Merge pull request #2455 from terascope/teraslice-improvements-jan-2021

fixing issue when getting job list from k8s This is meant to address #2438
terascope · Jan 28, 2021 · c3938ab · c3938ab
2 parents f3b12b6 + 6ed0830
commit c3938ab
Show file tree

Hide file tree

Showing 12 changed files with 61 additions and 35 deletions.
diff --git a/docs/configuration/clustering-k8s.md b/docs/configuration/clustering-k8s.md
@@ -6,6 +6,8 @@ sidebar_label: Kubernetes Clustering
 Teraslice supports the use of Kubernetes as a cluster manager. The following
 versions of Kuberenetes have been used:
 
+* `1.19.*`
+* `1.18.*`
 * `1.17.*`
 * `1.16.*`
 * `1.13.2`
@@ -374,7 +376,9 @@ cd examples/k8s
 export NAMESPACE=ts-dev1
 export TERASLICE_K8S_IMAGE=teraslice-k8sdev:1
 export TERASLICE_MODE=minikube
-minikube start --memory 4096 --cpus 4
+# Set the version of Kubernetes you want to run
+export KUBERNETES_VERSION=v1.19.7
+minikube start --memory 4096 --cpus 4 --kubernetes-version=${KUBERNETES_VERSION}
 eval $(minikube docker-env)
 make build
 make setup-all

diff --git a/examples/k8s/example-job-labels.json b/examples/k8s/example-job-labels.json
@@ -18,13 +18,10 @@
             "_op": "example-op"
         },
         {
-            "_op": "elasticsearch_index_selector",
+            "_op": "elasticsearch_bulk",
+            "size": 5000,
             "index": "terak8s-example-data",
             "type": "events"
-        },
-        {
-            "_op": "elasticsearch_bulk",
-            "size": 5000
         }
     ]
 }
diff --git a/examples/k8s/example-job-resource.json b/examples/k8s/example-job-resource.json
@@ -17,13 +17,10 @@
             "_op": "example-op"
         },
         {
-            "_op": "elasticsearch_index_selector",
+            "_op": "elasticsearch_bulk",
+            "size": 5000,
             "index": "terak8s-example-data",
             "type": "events"
-        },
-        {
-            "_op": "elasticsearch_bulk",
-            "size": 5000
         }
     ]
 }
diff --git a/examples/k8s/example-job-targets.json b/examples/k8s/example-job-targets.json
@@ -21,13 +21,10 @@
             "_op": "example-op"
         },
         {
-            "_op": "elasticsearch_index_selector",
+            "_op": "elasticsearch_bulk",
+            "size": 5000,
             "index": "terak8s-example-data",
             "type": "events"
-        },
-        {
-            "_op": "elasticsearch_bulk",
-            "size": 5000
         }
     ]
 }
diff --git a/examples/k8s/example-job-volume.json b/examples/k8s/example-job-volume.json
@@ -21,13 +21,10 @@
             "_op": "example-op"
         },
         {
-            "_op": "elasticsearch_index_selector",
+            "_op": "elasticsearch_bulk",
+            "size": 5000,
             "index": "terak8s-example-data",
             "type": "events"
-        },
-        {
-            "_op": "elasticsearch_bulk",
-            "size": 5000
         }
     ]
 }
diff --git a/examples/k8s/example-job-with-image.json b/examples/k8s/example-job-with-image.json
@@ -15,13 +15,10 @@
         {
             "_op": "example-op"
         },
-        {
-            "_op": "elasticsearch_index_selector",
-            "index": "terak8s-example-data",
-            "type": "events"
-        },
         {
             "_op": "elasticsearch_bulk",
+            "index": "terak8s-example-data",
+            "type": "events",
             "size": 5000
         }
     ]

diff --git a/examples/k8s/example-job.json b/examples/k8s/example-job.json
@@ -19,13 +19,10 @@
             "ms": 30000
         },
         {
-            "_op": "elasticsearch_index_selector",
+            "_op": "elasticsearch_bulk",
+            "size": 5000,
             "index": "terak8s-example-data",
             "type": "events"
-        },
-        {
-            "_op": "elasticsearch_bulk",
-            "size": 5000
         }
     ]
 }
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice-workspace",
     "displayName": "Teraslice",
-    "version": "0.71.2",
+    "version": "0.71.3",
     "private": true,
     "homepage": "https://github.com/terascope/teraslice",
     "bugs": {

diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/index.js b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/index.js
@@ -1,12 +1,13 @@
 'use strict';
 
 const {
-    TSError, logError, get, cloneDeep
+    TSError, logError, get, cloneDeep, pRetry
 } = require('@terascope/utils');
 const { makeLogger } = require('../../../../../workers/helpers/terafoundation');
 const K8sResource = require('./k8sResource');
 const k8sState = require('./k8sState');
 const K8s = require('./k8s');
+const { getRetryConfig } = require('./utils');
 
 /*
  Execution Life Cycle for _status
@@ -127,7 +128,9 @@ module.exports = function kubernetesClusterBackend(context, clusterMasterServer)
         // because they are not on the schema.  So I do this k8s API call
         // instead.
         const selector = `app.kubernetes.io/component=execution_controller,teraslice.terascope.io/jobId=${execution.job_id}`;
-        const jobs = await k8s.list(selector, 'jobs');
+        const jobs = await pRetry(
+            () => k8s.nonEmptyList(selector, 'jobs'), getRetryConfig
+        );
         execution.k8sName = jobs.items[0].metadata.name;
         execution.k8sUid = jobs.items[0].metadata.uid;
 

diff --git a/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8s.js b/packages/teraslice/lib/cluster/services/cluster/backends/kubernetes/k8s.js
@@ -197,6 +197,21 @@ class K8s {
         return response.body;
     }
 
+    async nonEmptyList(selector, objType) {
+        const jobs = await this.list(selector, objType);
+        if (jobs.items.length === 1) {
+            return jobs;
+        } if (jobs.items.length === 0) {
+            const msg = `Teraslice ${objType} matching the following selector was not found: ${selector} (retriable)`;
+            this.logger.warn(msg);
+            throw new TSError(msg, { retryable: true });
+        } else {
+            throw new TSError(`Unexpected number of Teraslice ${objType}s matching the following selector: ${selector}`, {
+                retryable: true
+            });
+        }
+    }
+
     /**
      * posts manifest to k8s
      * @param  {Object} manifest     service manifest

diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json
@@ -1,7 +1,7 @@
 {
     "name": "teraslice",
     "displayName": "Teraslice",
-    "version": "0.71.2",
+    "version": "0.71.3",
     "description": "Distributed computing platform for processing JSON data",
     "homepage": "https://github.com/terascope/teraslice#readme",
     "bugs": {

diff --git a/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8s-spec.js b/packages/teraslice/test/lib/cluster/services/cluster/backends/kubernetes/k8s-spec.js
@@ -104,6 +104,28 @@ describe('k8s', () => {
         });
     });
 
+    describe('->nonEmptyList', () => {
+        it('can get list with one item', async () => {
+            nock(_url)
+                .get('/apis/batch/v1/namespaces/default/jobs/')
+                .query({ labelSelector: 'app=teraslice' })
+                .reply(200, { items: ['one'] });
+
+            const jobs = await k8s.nonEmptyList('app=teraslice', 'jobs');
+            expect(jobs.items[0]).toEqual('one');
+        });
+
+        it('throws with an empty list', async () => {
+            nock(_url)
+                .get('/apis/batch/v1/namespaces/default/jobs/')
+                .query({ labelSelector: 'app=teraslice' })
+                .reply(200, { items: [] });
+
+            await expect(k8s.nonEmptyList('app=teraslice', 'jobs'))
+                .rejects.toThrowError('Teraslice jobs matching the following selector was not found: app=teraslice (retriable)');
+        });
+    });
+
     describe('->post', () => {
         it('can post a service', async () => {
             nock(_url, { encodedQueryParams: true })