Merge pull request #60 from srinandan/issue56

grant permissions to SA

Dockerfile

@@ -4,6 +4,7 @@ ADD ./apiclient /go/src/integrationcli/apiclient
 ADD ./client /go/src/integrationcli/client
 ADD ./cmd /go/src/integrationcli/cmd
 ADD ./cloudkms /go/src/integrationcli/cloudkms
+ADD ./secmgr /go/src/integrationcli/secmgr
 
 COPY main.go /go/src/integrationcli/main.go
 COPY go.mod go.sum /go/src/integrationcli/

Dockerfile.builder

@@ -4,6 +4,7 @@ ADD ./apiclient /go/src/integrationcli/apiclient
 ADD ./client /go/src/integrationcli/client
 ADD ./cmd /go/src/integrationcli/cmd
 ADD ./cloudkms /go/src/integrationcli/cloudkms
+ADD ./secmgr /go/src/integrationcli/secmgr
 
 COPY main.go /go/src/integrationcli/main.go
 COPY go.mod go.sum /go/src/integrationcli/

README.md

@@ -71,6 +71,96 @@ A sample customer cloud builder has been created to demonstrate automation of de
 
 This sample [cloud-build](./test/cloudbuild.yaml) demonstrates how to automate Integration version creation and publishing. Find more samples in this [repo](https://github.com/srinandan/integration-cicd)
 
+## Creating Integration Connectors
+
+`integrationcli` can be used to create [Integration Connectors](https://cloud.google.com/integration-connectors/docs). There are two types of Integration Connectors:
+
+### Google Managed Applications
+
+Google managed applications include systems like BigQuery, PubSub, Cloud SQL etc. To create a connection for such systems, author a json file like this:
+
+```yaml
+{
+    "description": "This is a sample",
+    "connectorDetails": {
+        "name": "pubsub", ## type of the connector
+        "version": 1 ## version is always 1
+    },
+    "configVariables": [ ## these values are specific to each connector type. this example is for pubsub
+        {
+            "key": "project_id",
+            "stringValue": "your-project-id" ## replace this 
+        },
+        {
+            "key": "topic_id",
+            "stringValue": "mytopic"
+        }
+    ],    
+    "serviceAccount": "sa-name@your-project-id.iam.gserviceaccount.com" ## replace this with a SA that has access to the application
+}
+```
+
+Then execute via `integrationcli` like this:
+
+```sh
+integrationcli connectors create -n name-of-the-connector -f ./test/pub_sub_connection.json
+```
+
+**NOTES:** 
+
+* This command assumes the token is cached, otherwise pass the token via `-t`
+* For PubSub, `integrationcli` adds the IAM permissions for the service account to publish to the topic
+
+### Third Party Applications
+
+Third party application include connectors like Salesforce, Service Now, etc. To create a connection for such systems, author a json file like this:
+
+```yaml
+{
+    "description": "SFTP Test for demo",
+    "connectorDetails": {
+        "name": "sftp", ## type of the connector
+        "version": 1 ## version is always 1
+    },    
+    "configVariables": [ ## these values are specific to each connector type. this example is for sftp
+        {
+            "key": "remote_host",
+            "stringValue": "example.net"
+        },
+        {
+            "key": "remote_port",
+            "stringValue": "22"
+        }
+    ],
+    "authConfig": {
+        "authType": "USER_PASSWORD",
+        "userPassword": {
+            "username": "demo",
+            "passwordDetails": {
+                "secretName": "sftp-demo", ## this secret is provisioned if it doesn't already exist
+                "reference": "./test/password.txt" ## this file contains the data/contents to put in secret manager
+            }
+        }
+    }
+}
+```
+
+If the connector depends on secret manager, `integrationcli` can create the Secret Manager secret if it is not already provisioned.
+
+Then execute via `integrationcli` like this:
+
+```sh
+integrationcli connectors create -n name-of-the-connector -f ./test/sftp_connection.json
+```
+
+NOTE: This command assumes the token is cached, otherwise pass the token via `-t`
+
+### Examples
+
+* [Big Query](./test/bq_connection.json)
+* [Service Now](./test/servicenow_connection.json)
+* [Salesforce](./test/salesforce_connections.json)
+
 ___
 
 ## Support

apiclient/iam.go

@@ -50,27 +50,10 @@ type setIamPolicy struct {
 	Policy iamPolicy `json:"policy,omitempty"`
 }
 
-// SetIAMPermission set permissions for a member on a connection
-func SetIAMPermission(name string, memberName string, iamRole string, memberType string) (err error) {
-	var role string
-
-	switch iamRole {
-	case "admin":
-		role = "roles/connectors.admin"
-	case "invoker":
-		role = "roles/connectors.invoker"
-	case "viewer":
-		role = "roles/connectors.viewer"
-	default: //assume this is a custom role definition
-		re := regexp.MustCompile(`projects\/([a-zA-Z0-9_-]+)\/roles\/([a-zA-Z0-9_-]+)`)
-		result := re.FindString(iamRole)
-		if result == "" {
-			return fmt.Errorf("custom role must be of the format projects/{project-id}/roles/{role-name}")
-		}
-		role = iamRole
-	}
+// setIAMPermission set permissions for a member
+func setIAMPermission(endpoint string, name string, memberName string, role string, memberType string) (err error) {
 
-	u, _ := url.Parse(GetBaseConnectorURL())
+	u, _ := url.Parse(endpoint)
 	u.Path = path.Join(u.Path, name+":getIamPolicy")
 	getIamPolicyBody, err := HttpClient(false, u.String())
 	if err != nil {
@@ -103,7 +86,7 @@ func SetIAMPermission(name string, memberName string, iamRole string, memberType
 		getIamPolicy.Bindings = append(getIamPolicy.Bindings, binding)
 	}
 
-	u, _ = url.Parse(GetBaseConnectorURL())
+	u, _ = url.Parse(endpoint)
 	u.Path = path.Join(u.Path, name+":setIamPolicy")
 
 	setIamPolicy := setIamPolicy{}
@@ -119,3 +102,34 @@ func SetIAMPermission(name string, memberName string, iamRole string, memberType
 
 	return err
 }
+
+// SetConnectorIAMPermission set permissions for a member on a connection
+func SetConnectorIAMPermission(name string, memberName string, iamRole string, memberType string) (err error) {
+	var role string
+
+	switch iamRole {
+	case "admin":
+		role = "roles/connectors.admin"
+	case "invoker":
+		role = "roles/connectors.invoker"
+	case "viewer":
+		role = "roles/connectors.viewer"
+	default: //assume this is a custom role definition
+		re := regexp.MustCompile(`projects\/([a-zA-Z0-9_-]+)\/roles\/([a-zA-Z0-9_-]+)`)
+		result := re.FindString(iamRole)
+		if result == "" {
+			return fmt.Errorf("custom role must be of the format projects/{project-id}/roles/{role-name}")
+		}
+		role = iamRole
+	}
+
+	return setIAMPermission(GetBaseConnectorURL(), name, memberName, role, memberType)
+}
+
+// SetPubSubIAMPermission set permissions for a SA on a topic
+func SetPubSubIAMPermission(project string, topic string, memberName string) (err error) {
+	var endpoint = fmt.Sprintf("https://pubsub.googleapis.com/v1/projects/%s/topics", project)
+	const memberType = "serviceAccount"
+	const role = "roles/pubsub.publisher"
+	return setIAMPermission(endpoint, topic, memberName, role, memberType)
+}

client/connections/connectors.go

@@ -15,13 +15,189 @@
 package connections
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 
+	"github.com/apigee/apigeecli/clilog"
 	"github.com/srinandan/integrationcli/apiclient"
+	"github.com/srinandan/integrationcli/secmgr"
 )
 
+type connectionRequest struct {
+	Labels             *map[string]string   `json:"labels,omitempty"`
+	Description        *string              `json:"description,omitempty"`
+	ConnectorDetails   *connectorDetails    `json:"connectorDetails,omitempty"`
+	ConnectorVersion   *string              `json:"connectorVersion,omitempty"`
+	ConfigVariables    *[]configVar         `json:"configVariables,omitempty"`
+	LockConfig         *lockConfig          `json:"lockConfig,omitempty"`
+	DestinationConfigs *[]destinationConfig `json:"deatinationConfigs,omitempty"`
+	AuthConfig         *authConfig          `json:"authConfig,omitempty"`
+	ServiceAccount     *string              `json:"serviceAccount,omitempty"`
+	Suspended          *bool                `json:"suspended,omitempty"`
+	NodeConfig         *nodeConfig          `json:"nodeConfig,omitempty"`
+}
+
+type authConfig struct {
+	AuthType                string                   `json:"authType,omitempty"`
+	UserPassword            *userPassword            `json:"userPassword,omitempty"`
+	Oauth2JwtBearer         *oauth2JwtBearer         `json:"oauth2JwtBearer,omitempty"`
+	Oauth2ClientCredentials *oauth2ClientCredentials `json:"oauth2ClientCredentials,omitempty"`
+	SshPublicKey            *sshPublicKey            `json:"sshPublicKey,omitempty"`
+}
+
+type lockConfig struct {
+	Locked bool   `json:"locked,omitempty"`
+	Reason string `json:"reason,omitempty"`
+}
+
+type connectorDetails struct {
+	Name    string `json:"name,omitempty"`
+	Version int    `json:"version,omitempty"`
+}
+
+type configVar struct {
+	Key           string         `json:"key,omitempty"`
+	IntValue      *string        `json:"intValue,omitempty"`
+	BoolValue     *bool          `json:"boolValue,omitempty"`
+	StringValue   *string        `json:"stringValue,omitempty"`
+	SecretValue   *secret        `json:"secretValue,omitempty"`
+	SecretDetails *secretDetails `json:"secretDetails,omitempty"`
+}
+
+type destinationConfig struct {
+	Key          string        `json:"key,omitempty"`
+	Destinations []destination `json:"destinations,omitempty"`
+}
+
+type userPassword struct {
+	Username        string         `json:"username,omitempty"`
+	Password        *secret        `json:"password,omitempty"`
+	PasswordDetails *secretDetails `json:"passwordDetails,omitempty"`
+}
+
+type oauth2JwtBearer struct {
+	ClientKey        *secret        `json:"clientKey,omitempty"`
+	ClientKeyDetails *secretDetails `json:"clientKeyDetails,omitempty"`
+	JwtClaims        jwtClaims      `json:"jwtClaims,omitempty"`
+}
+
+type oauth2ClientCredentials struct {
+	ClientId            string         `json:"clientId,omitempty"`
+	ClientSecret        *secret        `json:"clientSecret,omitempty"`
+	ClientSecretDetails *secretDetails `json:"clientSecretDetails,omitempty"`
+}
+
+type secret struct {
+	SecretVersion string `json:"secretVersion,omitempty"`
+}
+
+type secretDetails struct {
+	SecretName string `json:"secretName,omitempty"`
+	Reference  string `json:"reference,omitempty"`
+}
+
+type jwtClaims struct {
+	Issuer   string `json:"issuer,omitempty"`
+	Subject  string `json:"subject,omitempty"`
+	Audience string `json:"audience,omitempty"`
+}
+
+type sshPublicKey struct {
+	Username          string `json:"username,omitempty"`
+	Password          secret `json:"password,omitempty"`
+	SshClientCert     secret `json:"sshClientCert,omitempty"`
+	CertType          string `json:"certType,omitempty"`
+	SslClientCertPass secret `json:"sslClientCertPass,omitempty"`
+}
+
+type destination struct {
+	Port              int    `json:"port,omitempty"`
+	ServiceAttachment string `json:"serviceAttachment,omitempty"`
+	Host              string `json:"host,omitempty"`
+}
+
+type nodeConfig struct {
+	MinNodeCount string `json:"minNodeCount,omitempty"`
+	MaxNodeCount string `json:"maxNodeCount,omitempty"`
+}
+
+// Create
+func Create(name string, content []byte, grantPermission bool) (respBody []byte, err error) {
+
+	var secretVersion string
+
+	c := connectionRequest{}
+	if err = json.Unmarshal(content, &c); err != nil {
+		return nil, err
+	}
+
+	// check if permissions need to be set
+	if grantPermission && *c.ServiceAccount != "" {
+		switch c.ConnectorDetails.Name {
+		case "pubsub":
+			var projectId, topicName string
+
+			for _, configVar := range *c.ConfigVariables {
+				if configVar.Key == "project_id" {
+					projectId = *configVar.StringValue
+				}
+				if configVar.Key == "topic_id" {
+					topicName = *configVar.StringValue
+				}
+			}
+
+			if projectId == "" || topicName == "" {
+				return nil, fmt.Errorf("projectId or topicName was not set")
+			}
+
+			if err = apiclient.SetPubSubIAMPermission(projectId, topicName, *c.ServiceAccount); err != nil {
+				clilog.Warning.Printf("Unable to update permissions for the service account: %v\n", err)
+			}
+		case "bigquery":
+			clilog.Warning.Println("Updating service account permissions for BQ is not supported")
+		}
+	}
+
+	c.ConnectorVersion = new(string)
+	*c.ConnectorVersion = fmt.Sprintf("projects/%s/locations/global/providers/gcp/connectors/%s/versions/%d",
+		apiclient.GetProjectID(), c.ConnectorDetails.Name, c.ConnectorDetails.Version)
+
+	//remove the element
+	c.ConnectorDetails = nil
+
+	//handle secrets for username
+	if c.AuthConfig != nil && c.AuthConfig.UserPassword.PasswordDetails != nil {
+		payload, err := readSecretFile(c.AuthConfig.UserPassword.PasswordDetails.Reference)
+		if err != nil {
+			return nil, err
+		}
+
+		if secretVersion, err = secmgr.Create(apiclient.GetProjectID(), c.AuthConfig.UserPassword.PasswordDetails.SecretName, payload); err != nil {
+			return nil, err
+		}
+		c.AuthConfig.UserPassword.Password = new(secret)
+		c.AuthConfig.UserPassword.Password.SecretVersion = secretVersion
+		c.AuthConfig.UserPassword.PasswordDetails = nil //clean the input
+	}
+
+	u, _ := url.Parse(apiclient.GetBaseConnectorURL())
+	q := u.Query()
+	q.Set("connectionId", name)
+	u.RawQuery = q.Encode()
+
+	if content, err = json.Marshal(c); err != nil {
+		return nil, err
+	}
+
+	respBody, err = apiclient.HttpClient(apiclient.GetPrintOutput(), u.String(), string(content))
+	return respBody, err
+}
+
 // Delete
 func Delete(name string) (respBody []byte, err error) {
 	u, _ := url.Parse(apiclient.GetBaseConnectorURL())
@@ -63,3 +239,15 @@ func List(pageSize int, pageToken string, filter string, orderBy string) (respBo
 	respBody, err = apiclient.HttpClient(apiclient.GetPrintOutput(), u.String())
 	return respBody, err
 }
+
+func readSecretFile(name string) (payload []byte, err error) {
+	if _, err := os.Stat(name); os.IsNotExist(err) {
+		return nil, err
+	}
+
+	content, err := ioutil.ReadFile(name)
+	if err != nil {
+		return nil, err
+	}
+	return content, nil
+}