MM-16216 Add the ability to create clusters in existing subnets (#32)

* Add the ability to create clusters in existing subnets

* Refactor AWS client to be passed through supervisor

* Move logging of subnets on start-up to init log statement

* Only add private kops args if using private subnets

* Update tests

* Update internal/provisioner/kops_provisioner.go

Co-Authored-By: Gabe Jackson <gabe@coffeepowered.co>

cmd/cloud/server.go

@@ -33,6 +33,8 @@ func init() {
 	serverCmd.PersistentFlags().String("state-store", "dev.cloud.mattermost.com", "The S3 bucket used to store cluster state.")
 	serverCmd.PersistentFlags().String("certificate-aws-arn", "", "The certificate ARN from AWS. Generated in the certificate manager console.")
 	serverCmd.PersistentFlags().String("route53-id", "", "The route 53 hosted zone ID used for mattermost DNS records.")
+	serverCmd.PersistentFlags().String("private-subnets", "", "The private subnet IDs to use on AWS.")
+	serverCmd.PersistentFlags().String("public-subnets", "", "The public subnet IDs to use on AWS.")
 	serverCmd.PersistentFlags().Int("poll", 30, "The interval in seconds to poll for background work.")
 	serverCmd.PersistentFlags().Bool("debug", false, "Whether to output debug logs.")
 	serverCmd.MarkPersistentFlagRequired("route53-id")
@@ -71,6 +73,9 @@ var serverCmd = &cobra.Command{
 
 		s3StateStore, _ := command.Flags().GetString("state-store")
 		certificateSslARN, _ := command.Flags().GetString("certificate-aws-arn")
+		privateSubnetIds, _ := command.Flags().GetString("private-subnets")
+		publicSubnetIds, _ := command.Flags().GetString("public-subnets")
+		route53ZoneID, _ := command.Flags().GetString("route53-id")
 
 		wd, err := os.Getwd()
 		if err != nil {
@@ -83,13 +88,18 @@ var serverCmd = &cobra.Command{
 			"state-store":       s3StateStore,
 			"aws-arn":           certificateSslARN,
 			"working-directory": wd,
+			"private-subents":   privateSubnetIds,
+			"public-subnets":    publicSubnetIds,
+			"route53-id":        route53ZoneID,
 		}).Info("Starting Mattermost Provisioning Server")
 
 		// Setup the provisioner for actually effecting changes to clusters.
 		kopsProvisioner := provisioner.NewKopsProvisioner(
 			clusterRootDir,
 			s3StateStore,
 			certificateSslARN,
+			privateSubnetIds,
+			publicSubnetIds,
 			logger,
 		)
 
@@ -100,10 +110,9 @@ var serverCmd = &cobra.Command{
 		if poll == 0 {
 			logger.WithField("poll", poll).Info("Scheduler is disabled")
 		}
-		route53ZoneID, _ := command.Flags().GetString("route53-id")
 		supervisor := supervisor.NewScheduler(
 			supervisor.MultiDoer{
-				supervisor.NewClusterSupervisor(sqlStore, kopsProvisioner, instanceID, logger),
+				supervisor.NewClusterSupervisor(sqlStore, kopsProvisioner, aws.New(route53ZoneID), instanceID, logger),
 				supervisor.NewInstallationSupervisor(sqlStore, kopsProvisioner, aws.New(route53ZoneID), instanceID, logger),
 				supervisor.NewClusterInstallationSupervisor(sqlStore, kopsProvisioner, instanceID, logger),
 			},

go.sum

@@ -15,6 +15,7 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.19.41 h1:veutzvQP/lOmYmtX26S9mTFJLO6sp7/UsxFcCjglu4A=
 github.com/aws/aws-sdk-go v1.19.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.19.49 h1:GUlenK625g5iKrIiRcqRS/CvPMLc8kZRtMxXuXBhFx4=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=

internal/provisioner/kops_provisioner.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"strings"
 	"time"
 
 	mmv1alpha1 "github.com/mattermost/mattermost-operator/pkg/apis/mattermost/v1alpha1"
@@ -15,6 +16,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/mattermost/mattermost-cloud/internal/model"
+	"github.com/mattermost/mattermost-cloud/internal/tools/aws"
 	"github.com/mattermost/mattermost-cloud/internal/tools/k8s"
 	"github.com/mattermost/mattermost-cloud/internal/tools/kops"
 	"github.com/mattermost/mattermost-cloud/internal/tools/terraform"
@@ -25,15 +27,19 @@ type KopsProvisioner struct {
 	clusterRootDir    string
 	s3StateStore      string
 	certificateSslARN string
+	privateSubnetIds  string
+	publicSubnetIds   string
 	logger            log.FieldLogger
 }
 
 // NewKopsProvisioner creates a new KopsProvisioner.
-func NewKopsProvisioner(clusterRootDir, s3StateStore, certificateSslARN string, logger log.FieldLogger) *KopsProvisioner {
+func NewKopsProvisioner(clusterRootDir, s3StateStore, certificateSslARN, privateSubnetIds, publicSubnetIds string, logger log.FieldLogger) *KopsProvisioner {
 	return &KopsProvisioner{
 		clusterRootDir:    clusterRootDir,
 		s3StateStore:      s3StateStore,
 		certificateSslARN: certificateSslARN,
+		privateSubnetIds:  privateSubnetIds,
+		publicSubnetIds:   publicSubnetIds,
 		logger:            logger,
 	}
 }
@@ -58,7 +64,7 @@ func (provisioner *KopsProvisioner) PrepareCluster(cluster *model.Cluster) (bool
 }
 
 // CreateCluster creates a cluster using kops and terraform.
-func (provisioner *KopsProvisioner) CreateCluster(cluster *model.Cluster) error {
+func (provisioner *KopsProvisioner) CreateCluster(cluster *model.Cluster, aws aws.AWS) error {
 	kopsMetadata, err := model.NewKopsMetadata(cluster.ProvisionerMetadata)
 	if err != nil {
 		return errors.Wrap(err, "failed to parse provisioner metadata")
@@ -102,7 +108,7 @@ func (provisioner *KopsProvisioner) CreateCluster(cluster *model.Cluster) error
 		return err
 	}
 	defer kops.Close()
-	err = kops.CreateCluster(kopsMetadata.Name, cluster.Provider, clusterSize, awsMetadata.Zones)
+	err = kops.CreateCluster(kopsMetadata.Name, cluster.Provider, clusterSize, awsMetadata.Zones, provisioner.privateSubnetIds, provisioner.publicSubnetIds)
 	if err != nil {
 		return err
 	}
@@ -151,6 +157,18 @@ func (provisioner *KopsProvisioner) CreateCluster(cluster *model.Cluster) error
 		return err
 	}
 
+	// Set the ELB tags for the public subnets
+	if provisioner.publicSubnetIds != "" {
+		subnets := strings.Split(provisioner.publicSubnetIds, ",")
+		for _, subnet := range subnets {
+			logger.WithField("name", kopsMetadata.Name).Infof("Tagging subnet %s", subnet)
+			err = aws.TagResource(subnet, fmt.Sprintf("kubernetes.io/cluster/%s", kopsMetadata.Name), "shared", logger)
+			if err != nil {
+				return errors.Wrap(err, "failed to tag subnet")
+			}
+		}
+	}
+
 	logger.WithField("name", kopsMetadata.Name).Info("Successfully deployed kubernetes")
 
 	// Begin deploying the mattermost operator.
@@ -353,7 +371,7 @@ func (provisioner *KopsProvisioner) UpgradeCluster(cluster *model.Cluster) error
 }
 
 // DeleteCluster deletes a previously created cluster using kops and terraform.
-func (provisioner *KopsProvisioner) DeleteCluster(cluster *model.Cluster) error {
+func (provisioner *KopsProvisioner) DeleteCluster(cluster *model.Cluster, aws aws.AWS) error {
 	kopsMetadata, err := model.NewKopsMetadata(cluster.ProvisionerMetadata)
 	if err != nil {
 		return errors.Wrap(err, "failed to parse provisioner metadata")
@@ -417,6 +435,18 @@ func (provisioner *KopsProvisioner) DeleteCluster(cluster *model.Cluster) error
 		}
 	}
 
+	// Delete the ELB tags for the public subnets
+	if kopsMetadata.Name != "" && provisioner.publicSubnetIds != "" {
+		subnets := strings.Split(provisioner.publicSubnetIds, ",")
+		for _, subnet := range subnets {
+			logger.WithField("name", kopsMetadata.Name).Infof("Untagging subnet %s", subnet)
+			err = aws.UntagResource(subnet, fmt.Sprintf("kubernetes.io/cluster/%s", kopsMetadata.Name), "shared", logger)
+			if err != nil {
+				return errors.Wrap(err, "failed to untag subnet")
+			}
+		}
+	}
+
 	err = os.RemoveAll(outputDir)
 	if err != nil {
 		return errors.Wrap(err, "failed to clean up output directory")

internal/supervisor/cluster.go

@@ -2,6 +2,7 @@ package supervisor
 
 import (
 	"github.com/mattermost/mattermost-cloud/internal/model"
+	"github.com/mattermost/mattermost-cloud/internal/tools/aws"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -19,9 +20,9 @@ type clusterStore interface {
 // clusterProvisioner abstracts the provisioning operations required by the cluster supervisor.
 type clusterProvisioner interface {
 	PrepareCluster(cluster *model.Cluster) (bool, error)
-	CreateCluster(cluster *model.Cluster) error
+	CreateCluster(cluster *model.Cluster, aws aws.AWS) error
 	UpgradeCluster(cluster *model.Cluster) error
-	DeleteCluster(cluster *model.Cluster) error
+	DeleteCluster(cluster *model.Cluster, aws aws.AWS) error
 }
 
 // ClusterSupervisor finds clusters pending work and effects the required changes.
@@ -31,15 +32,17 @@ type clusterProvisioner interface {
 type ClusterSupervisor struct {
 	store       clusterStore
 	provisioner clusterProvisioner
+	aws         aws.AWS
 	instanceID  string
 	logger      log.FieldLogger
 }
 
 // NewClusterSupervisor creates a new ClusterSupervisor.
-func NewClusterSupervisor(store clusterStore, clusterProvisioner clusterProvisioner, instanceID string, logger log.FieldLogger) *ClusterSupervisor {
+func NewClusterSupervisor(store clusterStore, clusterProvisioner clusterProvisioner, aws aws.AWS, instanceID string, logger log.FieldLogger) *ClusterSupervisor {
 	return &ClusterSupervisor{
 		store:       store,
 		provisioner: clusterProvisioner,
+		aws:         aws,
 		instanceID:  instanceID,
 		logger:      logger,
 	}
@@ -115,7 +118,7 @@ func (s *ClusterSupervisor) transitionCluster(cluster *model.Cluster, logger log
 			}
 		}
 
-		err = s.provisioner.CreateCluster(cluster)
+		err = s.provisioner.CreateCluster(cluster, s.aws)
 		if err != nil {
 			logger.WithError(err).Error("Failed to create cluster")
 			return model.ClusterStateCreationFailed
@@ -135,7 +138,7 @@ func (s *ClusterSupervisor) transitionCluster(cluster *model.Cluster, logger log
 		return model.ClusterStateStable
 
 	case model.ClusterStateDeletionRequested:
-		err := s.provisioner.DeleteCluster(cluster)
+		err := s.provisioner.DeleteCluster(cluster, s.aws)
 		if err != nil {
 			logger.WithError(err).Error("Failed to delete cluster")
 			return model.ClusterStateDeletionFailed

internal/supervisor/cluster_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/mattermost/mattermost-cloud/internal/store"
 	"github.com/mattermost/mattermost-cloud/internal/supervisor"
 	"github.com/mattermost/mattermost-cloud/internal/testlib"
+	"github.com/mattermost/mattermost-cloud/internal/tools/aws"
 	"github.com/stretchr/testify/require"
 )
 
@@ -58,15 +59,15 @@ func (p *mockClusterProvisioner) PrepareCluster(cluster *model.Cluster) (bool, e
 	return true, nil
 }
 
-func (p *mockClusterProvisioner) CreateCluster(cluster *model.Cluster) error {
+func (p *mockClusterProvisioner) CreateCluster(cluster *model.Cluster, aws aws.AWS) error {
 	return nil
 }
 
 func (p *mockClusterProvisioner) UpgradeCluster(cluster *model.Cluster) error {
 	return nil
 }
 
-func (p *mockClusterProvisioner) DeleteCluster(cluster *model.Cluster) error {
+func (p *mockClusterProvisioner) DeleteCluster(cluster *model.Cluster, aws aws.AWS) error {
 	return nil
 }
 
@@ -75,7 +76,7 @@ func TestClusterSupervisorDo(t *testing.T) {
 		logger := testlib.MakeLogger(t)
 		mockStore := &mockClusterStore{}
 
-		supervisor := supervisor.NewClusterSupervisor(mockStore, &mockClusterProvisioner{}, "instanceID", logger)
+		supervisor := supervisor.NewClusterSupervisor(mockStore, &mockClusterProvisioner{}, &mockAWS{}, "instanceID", logger)
 		err := supervisor.Do()
 		require.NoError(t, err)
 
@@ -93,7 +94,7 @@ func TestClusterSupervisorDo(t *testing.T) {
 		mockStore.Cluster = mockStore.UnlockedClustersPendingWork[0]
 		mockStore.UnlockChan = make(chan interface{})
 
-		supervisor := supervisor.NewClusterSupervisor(mockStore, &mockClusterProvisioner{}, "instanceID", logger)
+		supervisor := supervisor.NewClusterSupervisor(mockStore, &mockClusterProvisioner{}, &mockAWS{}, "instanceID", logger)
 		err := supervisor.Do()
 		require.NoError(t, err)
 
@@ -118,7 +119,7 @@ func TestClusterSupervisorSupervise(t *testing.T) {
 		t.Run(tc.Description, func(t *testing.T) {
 			logger := testlib.MakeLogger(t)
 			sqlStore := store.MakeTestSQLStore(t, logger)
-			supervisor := supervisor.NewClusterSupervisor(sqlStore, &mockClusterProvisioner{}, "instanceID", logger)
+			supervisor := supervisor.NewClusterSupervisor(sqlStore, &mockClusterProvisioner{}, &mockAWS{}, "instanceID", logger)
 
 			cluster := &model.Cluster{
 				Provider: model.ProviderAWS,

internal/supervisor/installation.go

@@ -2,6 +2,7 @@ package supervisor
 
 import (
 	"github.com/mattermost/mattermost-cloud/internal/model"
+	"github.com/mattermost/mattermost-cloud/internal/tools/aws"
 	mmv1alpha1 "github.com/mattermost/mattermost-operator/pkg/apis/mattermost/v1alpha1"
 	log "github.com/sirupsen/logrus"
 )
@@ -36,26 +37,20 @@ type installationProvisioner interface {
 	GetClusterInstallationResource(cluster *model.Cluster, installation *model.Installation, clusterInstallation *model.ClusterInstallation) (*mmv1alpha1.ClusterInstallation, error)
 }
 
-// aws abstracts the aws client operations required by the installation supervisor.
-type aws interface {
-	CreateCNAME(dnsName string, dnsEndpoints []string, logger log.FieldLogger) error
-	DeleteCNAME(dnsName string, logger log.FieldLogger) error
-}
-
 // InstallationSupervisor finds installations pending work and effects the required changes.
 //
 // The degree of parallelism is controlled by a weighted semaphore, intended to be shared with
 // other clients needing to coordinate background jobs.
 type InstallationSupervisor struct {
 	store       installationStore
 	provisioner installationProvisioner
-	aws         aws
+	aws         aws.AWS
 	instanceID  string
 	logger      log.FieldLogger
 }
 
 // NewInstallationSupervisor creates a new InstallationSupervisor.
-func NewInstallationSupervisor(store installationStore, installationProvisioner installationProvisioner, aws aws, instanceID string, logger log.FieldLogger) *InstallationSupervisor {
+func NewInstallationSupervisor(store installationStore, installationProvisioner installationProvisioner, aws aws.AWS, instanceID string, logger log.FieldLogger) *InstallationSupervisor {
 	return &InstallationSupervisor{
 		store:       store,
 		provisioner: installationProvisioner,

internal/supervisor/installation_test.go

@@ -123,6 +123,14 @@ func (a *mockAWS) DeleteCNAME(dnsName string, logger log.FieldLogger) error {
 	return nil
 }
 
+func (a *mockAWS) TagResource(resourceID, key, value string, logger log.FieldLogger) error {
+	return nil
+}
+
+func (a *mockAWS) UntagResource(resourceID, key, value string, logger log.FieldLogger) error {
+	return nil
+}
+
 func TestInstallationSupervisorDo(t *testing.T) {
 	t.Run("no clusters pending work", func(t *testing.T) {
 		logger := testlib.MakeLogger(t)

internal/tools/aws/client.go

@@ -1,18 +1,36 @@
 package aws
 
-import "github.com/aws/aws-sdk-go/service/route53"
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/route53"
+	log "github.com/sirupsen/logrus"
+)
+
+// AWS interface for use by other packages.
+type AWS interface {
+	CreateCNAME(dnsName string, dnsEndpoints []string, logger log.FieldLogger) error
+	DeleteCNAME(dnsName string, logger log.FieldLogger) error
+	TagResource(resourceID, key, value string, logger log.FieldLogger) error
+	UntagResource(resourceID, key, value string, logger log.FieldLogger) error
+}
 
 // Client is a client for interacting with AWS resources.
 type Client struct {
 	hostedZoneID string
 	api          api
 }
 
+var _ AWS = &Client{}
+
 // api mocks out the AWS API calls for testing.
 type api interface {
-	getSessionClient() (*route53.Route53, error)
+	getRoute53Client() (*route53.Route53, error)
 	changeResourceRecordSets(*route53.Route53, *route53.ChangeResourceRecordSetsInput) (*route53.ChangeResourceRecordSetsOutput, error)
 	listResourceRecordSets(*route53.Route53, *route53.ListResourceRecordSetsInput) (*route53.ListResourceRecordSetsOutput, error)
+
+	getEC2Client() (*ec2.EC2, error)
+	tagResource(*ec2.EC2, *ec2.CreateTagsInput) (*ec2.CreateTagsOutput, error)
+	untagResource(*ec2.EC2, *ec2.DeleteTagsInput) (*ec2.DeleteTagsOutput, error)
 }
 
 // New returns a new AWS client.