Skip to content

Commit

Permalink
gchqgh-50 Integrating namespaces with graphs - work in progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@m29827
m29827 committed Sep 24, 2020
1 parent 5e7fe0a commit 45519e5
Show file tree
Hide file tree
Showing 21 changed files with 705 additions and 473 deletions.
1 change: 1 addition & 0 deletions infrastructure/lib/database/database.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export class Database extends cdk.Construct {

this._graphTable = new dynamo.Table(this, "GraphDynamoTable", {
partitionKey: { name: "releaseName", type: dynamo.AttributeType.STRING },
sortKey: { name: "namespaceName", type: dynamo.AttributeType.STRING },
billingMode: dynamo.BillingMode.PROVISIONED,
removalPolicy: cdk.RemovalPolicy.DESTROY
});
Expand Down
172 changes: 89 additions & 83 deletions infrastructure/lib/rest-api/kai-rest-api.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,12 @@ export class KaiRestApi extends cdk.Construct {
private readonly _restApi: api.RestApi;
private readonly _props: KaiRestApiProps;
private readonly _listCognitoUserPoolPolicyStatement: PolicyStatement;

private _addGraphQueue: sqs.Queue;
private _deleteGraphQueue: sqs.Queue;
private _getGraphsLambda: lambda.Function;
private _deleteGraphLambda: lambda.Function;

private _addNamespaceQueue: sqs.Queue;
private _deleteNamespaceQueue: sqs.Queue;
private readonly _getGraphsLambda: lambda.Function;
private readonly _deleteGraphLambda: lambda.Function;
private readonly _addGraphQueue: sqs.Queue = this.createQueue("AddGraphQueue", ADD_GRAPH_TIMEOUT);
private readonly _deleteGraphQueue: sqs.Queue = this.createQueue("DeleteGraphQueue", DELETE_GRAPH_TIMEOUT);
private readonly _addNamespaceQueue: sqs.Queue = this.createQueue("AddNamespaceQueue", ADD_NAMESPACE_TIMEOUT);
private readonly _deleteNamespaceQueue: sqs.Queue = this.createQueue("DeleteNamespaceQueue", DELETE_NAMESPACE_TIMEOUT);

constructor(scope: cdk.Construct, readonly id: string, props: KaiRestApiProps) {
super(scope, id);
Expand All @@ -65,114 +63,122 @@ export class KaiRestApi extends cdk.Construct {
this._lambdaTimeout = cdk.Duration.seconds(30);

// Create the API Resources
this.createGraphsResource();
this.createNamespacesResource();
}

private createGraphsResource(): void {
const graphsResource = this._restApi.root.addResource("graphs");
const graph = graphsResource.addResource("{graphName}");

// POST handlers
this._addGraphQueue = new sqs.Queue(this, "AddGraphQueue", {
visibilityTimeout: ADD_GRAPH_TIMEOUT
});
const graphResource = graphsResource.addResource("{graphName}");
const namespacesResource = this._restApi.root.addResource("namespaces");
const namespaceResource = namespacesResource.addResource("{namespaceName}");
const namespaceGraphsResource = namespaceResource.addResource("graphs");
const namespaceGraphResource = namespaceGraphsResource.addResource("{graphName}");

const addGraphLambdaEnvironment = {
sqs_queue_url: this.addGraphQueue.queueUrl,
// Graphs
const graphLambdaEnvironment = {
graph_table_name: this._props.graphTable.tableName,
user_pool_id: this._props.userPoolId
};

const addGraphLambda = this.createLambda("AddGraphHandler", "add_graph_request.handler", addGraphLambdaEnvironment);

addGraphLambda.addToRolePolicy(this._listCognitoUserPoolPolicyStatement);
const addGraphLambda = this.createAddGraphLambda(graphLambdaEnvironment);
this._getGraphsLambda = this.createGetGraphLambda(graphLambdaEnvironment);
this._deleteGraphLambda = this.createDeleteGraphLambda(graphLambdaEnvironment);

this._props.graphTable.grantReadWriteData(addGraphLambda);
this.addGraphQueue.grantSendMessages(addGraphLambda);
graphsResource.addMethod("POST", new api.LambdaIntegration(addGraphLambda), this._methodOptions);
const addGraphIntegration = new api.LambdaIntegration(addGraphLambda);
const deleteGraphIntegration = new api.LambdaIntegration(this._deleteGraphLambda);
const getGraphIntegration = new api.LambdaIntegration(this._getGraphsLambda);

// DELETE handlers
this._deleteGraphQueue = new sqs.Queue(this, "DeleteGraphQueue", {
visibilityTimeout: DELETE_GRAPH_TIMEOUT
});
graphsResource.addMethod("GET", getGraphIntegration, this._methodOptions);
namespaceGraphsResource.addMethod("GET", getGraphIntegration, this._methodOptions);
namespaceGraphResource.addMethod("GET", getGraphIntegration, this._methodOptions);
graphsResource.addMethod("POST", addGraphIntegration, this._methodOptions);
namespaceGraphResource.addMethod("DELETE", deleteGraphIntegration, this._methodOptions);

const deleteGraphLambdaEnvironment = {
sqs_queue_url: this.deleteGraphQueue.queueUrl,
graph_table_name: this._props.graphTable.tableName,
// Namespaces
const namespaceLambdaEnvironment = {
namespace_table_name: this._props.namespaceTable.tableName,
user_pool_id: this._props.userPoolId
};

this._deleteGraphLambda = this.createLambda("DeleteGraphHandler", "delete_graph_request.handler", deleteGraphLambdaEnvironment);

this._props.graphTable.grantReadWriteData(this._deleteGraphLambda);
this.deleteGraphQueue.grantSendMessages(this._deleteGraphLambda);
graph.addMethod("DELETE", new api.LambdaIntegration(this._deleteGraphLambda), this._methodOptions);

const getGraphLambdaEnvironment = {
graph_table_name: this._props.graphTable.tableName,
user_pool_id: this._props.userPoolId
};
const addNamespaceLambda = this.createAddNamespaceLambda(namespaceLambdaEnvironment);
const getNamespaceLambda = this.createGetNamespaceLambda(namespaceLambdaEnvironment);
const updateNamespaceLambda = this.createUpdateNamespaceLambda(namespaceLambdaEnvironment);
const deleteNamespaceLambda = this.createDeleteNamespaceLambda(namespaceLambdaEnvironment);

// GET handlers
this._getGraphsLambda = this.createLambda("GetGraphsHandler", "get_graph_request.handler", getGraphLambdaEnvironment);
const addNamespaceIntegration = new api.LambdaIntegration(addNamespaceLambda);
const deleteNamespaceIntegration = new api.LambdaIntegration(deleteNamespaceLambda);
const getNamespaceIntegration = new api.LambdaIntegration(getNamespaceLambda);
const updateNamespaceIntegration = new api.LambdaIntegration(updateNamespaceLambda);

this._props.graphTable.grantReadData(this._getGraphsLambda);
// Both GET and GET all are served by the same lambda
const getGraphIntegration = new api.LambdaIntegration(this._getGraphsLambda);
graphsResource.addMethod("GET", getGraphIntegration, this._methodOptions);
graph.addMethod("GET", getGraphIntegration, this._methodOptions);
namespacesResource.addMethod("POST", addNamespaceIntegration, this._methodOptions);
namespacesResource.addMethod("GET", getNamespaceIntegration, this._methodOptions);
namespaceResource.addMethod("GET", getNamespaceIntegration, this._methodOptions);
namespaceResource.addMethod("POST", updateNamespaceIntegration, this._methodOptions);
namespaceResource.addMethod("DELETE", deleteNamespaceIntegration, this._methodOptions);
}

private createQueue(id: string, visibilityTimeout: cdk.Duration): sqs.Queue {
return new sqs.Queue(this, id, {visibilityTimeout: visibilityTimeout});
}

private createNamespacesResource(): void {
const namespacesResource = this._restApi.root.addResource("namespaces");
const namespace = namespacesResource.addResource("{namespaceName}");

const namespaceLambdaEnvironment = {
namespace_table_name: this._props.namespaceTable.tableName,
user_pool_id: this._props.userPoolId
};

// Add
this._addNamespaceQueue = new sqs.Queue(this, "AddNamespaceQueue", {
visibilityTimeout: ADD_NAMESPACE_TIMEOUT
});

const addNamespaceLambdaEnvironment: {[key: string]: string} = Object.assign({}, namespaceLambdaEnvironment);
private createAddNamespaceLambda(environment: {[key: string]: string}): lambda.Function {
const addNamespaceLambdaEnvironment: {[key: string]: string} = Object.assign({}, environment);
addNamespaceLambdaEnvironment["sqs_queue_url"] = this.addNamespaceQueue.queueUrl;

const addNamespaceLambda = this.createLambda("AddNamespaceHandler", "add_namespace_request.handler", addNamespaceLambdaEnvironment);
addNamespaceLambda.addToRolePolicy(this._listCognitoUserPoolPolicyStatement);
this._props.namespaceTable.grantReadWriteData(addNamespaceLambda);
this.addNamespaceQueue.grantSendMessages(addNamespaceLambda);
namespacesResource.addMethod("POST", new api.LambdaIntegration(addNamespaceLambda), this._methodOptions);
return addNamespaceLambda;
}

// Get
const getNamespacesLambda = this.createLambda("GetNamespacesHandler", "get_namespace_request.handler", namespaceLambdaEnvironment);
private createGetNamespaceLambda(environment: {[key: string]: string}): lambda.Function {
const getNamespacesLambda = this.createLambda("GetNamespacesHandler", "get_namespace_request.handler", environment);
this._props.namespaceTable.grantReadData(getNamespacesLambda);
const getNamespaceIntegration = new api.LambdaIntegration(getNamespacesLambda);
namespacesResource.addMethod("GET", getNamespaceIntegration, this._methodOptions);
namespace.addMethod("GET", getNamespaceIntegration, this._methodOptions);
return getNamespacesLambda;
}

// Update
const updateNamespaceLambda = this.createLambda("UpdateNamespaceHandler", "update_namespace_request.handler", namespaceLambdaEnvironment);
private createUpdateNamespaceLambda(environment: {[key: string]: string}): lambda.Function {
const updateNamespaceLambda = this.createLambda("UpdateNamespaceHandler", "update_namespace_request.handler", environment);
updateNamespaceLambda.addToRolePolicy(this._listCognitoUserPoolPolicyStatement);
this._props.namespaceTable.grantReadWriteData(updateNamespaceLambda);
namespace.addMethod("POST", new api.LambdaIntegration(updateNamespaceLambda), this._methodOptions);

// Delete
this._deleteNamespaceQueue = new sqs.Queue(this, "DeleteNamespaceQueue", {
visibilityTimeout: DELETE_NAMESPACE_TIMEOUT
});
return updateNamespaceLambda;
}

const deleteNamespaceLambdaEnvironment: {[key: string]: string} = Object.assign({}, namespaceLambdaEnvironment);
private createDeleteNamespaceLambda(environment: {[key: string]: string}): lambda.Function {
const deleteNamespaceLambdaEnvironment: {[key: string]: string} = Object.assign({}, environment);
deleteNamespaceLambdaEnvironment["sqs_queue_url"] = this.deleteNamespaceQueue.queueUrl;

deleteNamespaceLambdaEnvironment["graph_table_name"] = this._props.graphTable.tableName;
const deleteNamespaceLambda = this.createLambda("DeleteNamespaceHandler", "delete_namespace_request.handler", deleteNamespaceLambdaEnvironment);
this._props.namespaceTable.grantReadWriteData(deleteNamespaceLambda);
this._props.graphTable.grantReadData(deleteNamespaceLambda);
this.deleteNamespaceQueue.grantSendMessages(deleteNamespaceLambda);
namespace.addMethod("DELETE", new api.LambdaIntegration(deleteNamespaceLambda), this._methodOptions);
return deleteNamespaceLambda;
}

private createAddGraphLambda(environment: {[key: string]: string}): lambda.Function {
const addGraphLambdaEnvironment: {[key: string]: string} = Object.assign({}, environment);
addGraphLambdaEnvironment["sqs_queue_url"] = this.addGraphQueue.queueUrl;
addGraphLambdaEnvironment["namespace_table_name"] = this._props.namespaceTable.tableName;

const addGraphLambda = this.createLambda("AddGraphHandler", "add_graph_request.handler", addGraphLambdaEnvironment);
addGraphLambda.addToRolePolicy(this._listCognitoUserPoolPolicyStatement);
this._props.graphTable.grantReadWriteData(addGraphLambda);
this._props.namespaceTable.grantReadData(addGraphLambda);
this.addGraphQueue.grantSendMessages(addGraphLambda);
return addGraphLambda;
}

private createGetGraphLambda(environment: {[key: string]: string}): lambda.Function {
const getGraphsLambda = this.createLambda("GetGraphsHandler", "get_graph_request.handler", environment);
this._props.graphTable.grantReadData(getGraphsLambda);
return getGraphsLambda;
}

private createDeleteGraphLambda(environment: {[key: string]: string}): lambda.Function {
const deleteGraphLambdaEnvironment: {[key: string]: string} = Object.assign({}, environment);
deleteGraphLambdaEnvironment["sqs_queue_url"] = this.deleteGraphQueue.queueUrl;

const deleteGraphLambda = this.createLambda("DeleteGraphHandler", "delete_graph_request.handler", deleteGraphLambdaEnvironment);
this._props.graphTable.grantReadWriteData(deleteGraphLambda);
this.deleteGraphQueue.grantSendMessages(deleteGraphLambda);
return deleteGraphLambda;
}

private createLambda(id: string, handler: string, environment: {[key: string]: string}): lambda.Function {
Expand Down
49 changes: 39 additions & 10 deletions infrastructure/lib/rest-api/lambdas/add_graph_request.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@
from botocore.exceptions import ClientError
from graph import Graph
import json
from namespace import Namespace
import os
import re
from user import User

graph = Graph()
namespace = Namespace()
user = User()

def is_graph_name_valid(graph_name):
Expand All @@ -30,17 +32,15 @@ def handler(event, context):
"statusCode": 400,
"body": "schema is a required field"
}
if "namespaceName" not in request_body or not namespace.is_namespace_name_valid(request_body["namespaceName"]):
return {
"statusCode": 400,
"body": "namespaceName is a required field which must be a valid DNS label"
}

graph_name = request_body["graphName"]
schema = request_body["schema"]

# Get variables from env
queue_url = os.getenv("sqs_queue_url")

# Convert graph name to lowercase
release_name = graph.format_graph_name(graph_name)

initial_status = "DEPLOYMENT_QUEUED"
namespace_name = request_body["namespaceName"]

administrators = []
requesting_user = user.get_requesting_cognito_user(event)
Expand All @@ -57,12 +57,40 @@ def handler(event, context):
}

try:
graph.create_graph(release_name, graph_name, initial_status, administrators)
namespace_record = namespace.get_namespace(namespace_name)
if not namespace_record["currentState"] == "DEPLOYED":
return {
"statusCode": 400,
"body": "Graph: {} can not be added to Namespace {} as the namespace is not in a DEPLOYED state".format(graph_name, namespace_name)
}
if requesting_user is not None and not namespace_record["isPublic"] and not requesting_user in namespace_record["administrators"]:
return {
"statusCode": 403,
"body": "User {} is not permitted to deploy a graph into namespace: {}".format(requesting_user, namespace_name)
}

except Exception as e:
return {
"statusCode": 400,
"body": "Could not create graph, namespace: " + namespace_name + " was not found"
}


# Get variables from env
queue_url = os.getenv("sqs_queue_url")

# Convert graph name to lowercase
release_name = graph.format_graph_name(graph_name)

initial_status = "DEPLOYMENT_QUEUED"

try:
graph.create_graph(release_name, graph_name, initial_status, administrators, namespace_name)
except ClientError as e:
if e.response['Error']['Code']=='ConditionalCheckFailedException':
return {
"statusCode": 400,
"body": "Graph release name " + release_name + " already exists as the lowercase conversion of " + graph_name + ". Graph names must be unique"
"body": "Graph release name: {} already exists in namespace: {}. Lowercase Graph names must be unique within a namespace.".format(release_name, namespace_name)
}
else:
return {
Expand All @@ -74,6 +102,7 @@ def handler(event, context):
message = {
"graphName": graph_name,
"releaseName": release_name,
"namespaceName": namespace_name,
"schema": schema,
"expectedStatus": initial_status
}
Expand Down
5 changes: 3 additions & 2 deletions infrastructure/lib/rest-api/lambdas/add_namespace_request.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import boto3
from botocore.exceptions import ClientError
import json
from namespace import Namespace
import os
import re

from botocore.exceptions import ClientError
from namespace import Namespace
from user import User

namespace = Namespace()
Expand Down
18 changes: 13 additions & 5 deletions infrastructure/lib/rest-api/lambdas/delete_graph_request.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ def handler(event, context):

# Check request is valid
graph_name = params["graphName"]
namespace_name = params["namespaceName"]

# Convert graph name to lowercase
release_name = graph.format_graph_name(graph_name)
Expand All @@ -26,30 +27,36 @@ def handler(event, context):
"body": "graphName is a required field"
}

if namespace_name is None:
return {
"statusCode": 400,
"body": "namespaceName is a required field"
}

try:
graph_record = graph.get_graph(release_name)
graph_record = graph.get_graph(release_name, namespace_name)
requesting_user = user.get_requesting_cognito_user(event)
if requesting_user and not requesting_user in graph_record["administrators"]:
return {
"statusCode": 403,
"body": "User: {} is not authorized to delete graph: {}".format(requesting_user, graph_name)
"body": "User: {} is not authorized to delete graph: {} in namespace: {}".format(requesting_user, graph_name, namespace_name)
}
except:
return {
"statusCode": 400,
"body": "Graph " + graph_name + " does not exist. It may have already been deleted"
"body": "Graph: {} in namespace: {} does not exist. It may have already been deleted".format(graph_name, namespace_name)
}

initial_status = "DELETION_QUEUED"

# Add Entry to table
try:
graph.update_graph(release_name, initial_status)
graph.update_graph(release_name, namespace_name, initial_status)
except ClientError as e:
if e.response['Error']['Code'] == 'ConditionalCheckFailedException':
return {
"statusCode": 400,
"body": "Graph " + graph_name + " does not exist. It may have already have been deleted"
"body": "Graph: {} in namespace: {} does not exist. It may have already been deleted".format(graph_name, namespace_name)
}
else:
return {
Expand All @@ -61,6 +68,7 @@ def handler(event, context):
message = {
"graphName": graph_name,
"releaseName": release_name,
"namespaceName": namespace_name,
"expectedStatus": initial_status
}

Expand Down
Loading

0 comments on commit 45519e5

Please sign in to comment.