Merge branch 'develop' of github.com:gchq/Kai into gchqgh-7-ui-develop

infrastructure/lib/rest-api/lambdas/add_graph_request.py

@@ -100,7 +100,8 @@ def handler(event, context):
         "graphName": graph_name,
         "releaseName": release_name,
         "schema": schema,
-        "expectedStatus": initial_status
+        "expectedStatus": initial_status,
+        "endpoints":{}
     }
 
     sqs = boto3.client("sqs")

infrastructure/lib/rest-api/lambdas/graph/__init__.py

@@ -51,13 +51,15 @@ def update_graph(self, release_name, status):
             ConditionExpression=boto3.dynamodb.conditions.Attr("releaseName").exists()
         )
 
+
     def create_graph(self, release_name, graph_name, status, administrators):      
         self.table.put_item(
             Item={
                 "graphName": graph_name,
                 "releaseName": release_name,
                 "currentState": status,
-                "administrators": administrators
+                "administrators": administrators,
+                "endpoints":{}
             },
             ConditionExpression=boto3.dynamodb.conditions.Attr("releaseName").not_exists()
         )

infrastructure/lib/workers/lambdas/add_graph.py

@@ -3,9 +3,11 @@
 import os
 import random
 import string
+import subprocess
 
 import boto3
 import kubernetes
+from kubernetes import KubernetesClient, KubeConfigurator, CommandHelper
 from graph import Graph
 
 logger = logging.getLogger()
@@ -16,6 +18,7 @@
 cluster_name = os.getenv("cluster_name")
 graph_table_name = os.getenv("graph_table_name")
 
+
 def generate_password(length=8):
     """
     Generates a random password of a given length
@@ -95,7 +98,17 @@ def create_values(graph_name, schema, security_groups):
         }
     }
 
-
+def get_endpoints(release_name, graph):
+    kubernetes_client = kubernetes.KubernetesClient(cluster_name)
+    alb_output = kubernetes_client.get_alb_endpoints(release_name)
+    alb_endpoints = alb_output.splitlines()
+    for line in alb_endpoints[1:]:
+        resource_details = line.split()
+        resource_name = resource_details[0] 
+        resource_address = resource_details[2]
+        graph.update_endpoints(resource_name, resource_address)
+
+
 def deploy_graph(helm_client, body, security_groups):
     """
     Deploys a Gaffer graph onto a Kubernetes cluster using the Gaffer
@@ -128,6 +141,8 @@ def deploy_graph(helm_client, body, security_groups):
     success = helm_client.install_chart(release_name, values=values_file)
 
     if success:
+        # capture endpoints
+        get_endpoints(release_name, graph)
         logger.info("Deployment of " + graph_name + " Succeeded")
         graph.update_status("DEPLOYED")
     else:
@@ -141,7 +156,7 @@ def handler(event, context):
     logger.info(event)
 
     helm_client = kubernetes.HelmClient(cluster_name)
-
+    
     # Get Security Groups
     eks = boto3.client("eks")
     cluster = eks.describe_cluster(name=cluster_name)

infrastructure/lib/workers/lambdas/graph/__init__.py

@@ -33,6 +33,26 @@ def check_status(self, expected_status):
 
         return status == expected_status
 
+
+    def update_endpoints(self, resource_name, resource_address):
+        """
+        Update graph with endpoints that get created by the application load balancer
+        """
+        self.table.update_item(
+            Key={
+                "releaseName": self.release_name
+            },
+            UpdateExpression = "SET endpoints.#resourceName = :resourceAddress",
+            ExpressionAttributeNames = { 
+                "#resourceName" : resource_name 
+                },
+            ExpressionAttributeValues = {
+                ":resourceAddress": resource_address
+            },
+            ConditionExpression = "attribute_not_exists(endpoints.#resourceName)"
+        )
+
+
     def update_status(self, status):
         """
         Updates the status of a Graph

infrastructure/lib/workers/lambdas/kubernetes/__init__.py

@@ -12,13 +12,15 @@ class CommandHelper:
     def run_command(cmd, release_name):
         succeeded=False
         try:
-            subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, cwd="/tmp")
+            cp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True, text=True, cwd="/tmp")
             succeeded=True
         except subprocess.CalledProcessError as err:
             logger.error("Error during excution of command: %s against release name: %s", cmd, release_name)
             logger.error(err.output)
-
-        return succeeded
+        if succeeded:
+            return(succeeded, cp.stdout)
+        else:
+            return succeeded
 
 
 class KubeConfigurator:
@@ -97,4 +99,12 @@ def __delete_volumes(self, release_name, selectors):
             cmd.append("--selector")
             cmd.append(selector)
 
-        CommandHelper.run_command(cmd, release_name)
+        CommandHelper.run_command(cmd, release_name)
+
+    def get_alb_endpoints(self, release_name):
+        cmd =[self.__KUBECTL_CMD, "get", "ing", "--kubeconfig", self.kubeconfig ]
+        output = CommandHelper.run_command(cmd, release_name)
+        if(type (output) == tuple):
+            return output[1]
+
+

infrastructure/test/e2e/integration.test.ts

@@ -91,6 +91,11 @@ describe("GET /graphs tests", () => {
                 clusterHelper.userTokens[user1].user.userName,
                 clusterHelper.userTokens[user3].user.userName
             ],
+            endpoints:{
+                "testgraph1-gaffer-api": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-gaffer-monitor": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-hdfs": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/)
+            },
             currentState: "DEPLOYED",
             releaseName: testGraph1.toLowerCase()
         });
@@ -107,6 +112,11 @@ describe("GET /graphs tests", () => {
                 clusterHelper.userTokens[user1].user.userName,
                 clusterHelper.userTokens[user3].user.userName
             ],
+            endpoints:{
+                "testgraph1-gaffer-api": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-gaffer-monitor": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-hdfs": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/)
+            },
             currentState: "DEPLOYED",
             releaseName: testGraph1.toLowerCase()
         });
@@ -125,6 +135,11 @@ describe("GET /graphs/{graphName} tests", () => {
                 clusterHelper.userTokens[user1].user.userName,
                 clusterHelper.userTokens[user3].user.userName
             ],
+            endpoints:{
+                "testgraph1-gaffer-api": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-gaffer-monitor": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-hdfs": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/)
+            },
             currentState: "DEPLOYED",
             releaseName: testGraph1.toLowerCase()
         });
@@ -140,6 +155,11 @@ describe("GET /graphs/{graphName} tests", () => {
                 clusterHelper.userTokens[user1].user.userName,
                 clusterHelper.userTokens[user3].user.userName
             ],
+            endpoints:{
+                "testgraph1-gaffer-api": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-gaffer-monitor": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/),
+                "testgraph1-hdfs": expect.stringMatching(/.*eu-west-1.elb.amazonaws.com/)
+            },
             currentState: "DEPLOYED",
             releaseName: testGraph1.toLowerCase()
         });