Merge pull request #1481 from wking/structured-workers

pkg/asset/machines/worker: Structured workers

data/data/aws/main.tf

@@ -72,9 +72,10 @@ module "dns" {
 module "vpc" {
   source = "./vpc"
 
-  cidr_block = "${var.machine_cidr}"
-  cluster_id = "${var.cluster_id}"
-  region     = "${var.aws_region}"
+  cidr_block         = "${var.machine_cidr}"
+  cluster_id         = "${var.cluster_id}"
+  region             = "${var.aws_region}"
+  availability_zones = "${distinct(concat(var.aws_master_availability_zones, var.aws_worker_availability_zones))}"
 
   tags = "${local.tags}"
 }

data/data/aws/variables-aws.tf

@@ -62,3 +62,8 @@ variable "aws_master_availability_zones" {
   type        = "list"
   description = "The availability zones in which to create the masters. The length of this list must match master_count."
 }
+
+variable "aws_worker_availability_zones" {
+  type        = "list"
+  description = "The availability zones to provision for workers.  Worker instances are created by the machine-API operator, but this variable controls their supporting infrastructure (subnets, routing, etc.)."
+}

data/data/aws/vpc/common.tf

@@ -1,19 +1,10 @@
 # Canonical internal state definitions for this module.
 # read only: only locals and data source definitions allowed. No resources or module blocks in this file
-data "aws_region" "current" {}
 
-// Fetch a list of available AZs
-data "aws_availability_zones" "azs" {
-  state = "available"
-}
-
-// Only reference data sources which are gauranteed to exist at any time (above) in this locals{} block
+// Only reference data sources which are guaranteed to exist at any time (above) in this locals{} block
 locals {
-  // List of possible AZs for each type of subnet
-  new_subnet_azs = "${data.aws_availability_zones.azs.names}"
-
   // How many AZs to create subnets in
-  new_az_count = "${length(local.new_subnet_azs)}"
+  new_az_count = "${length(var.availability_zones)}"
 
   // The VPC ID to use to build the rest of the vpc data sources
   vpc_id = "${aws_vpc.new_vpc.id}"

data/data/aws/vpc/outputs.tf

@@ -3,11 +3,11 @@ output "vpc_id" {
 }
 
 output "az_to_private_subnet_id" {
-  value = "${zipmap(local.new_subnet_azs, local.private_subnet_ids)}"
+  value = "${zipmap(var.availability_zones, local.private_subnet_ids)}"
 }
 
 output "az_to_public_subnet_id" {
-  value = "${zipmap(local.new_subnet_azs, local.public_subnet_ids)}"
+  value = "${zipmap(var.availability_zones, local.public_subnet_ids)}"
 }
 
 output "public_subnet_ids" {

data/data/aws/vpc/variables.tf

@@ -1,3 +1,8 @@
+variable "availability_zones" {
+  type        = "list"
+  description = "The availability zones in which to provision subnets."
+}
+
 variable "cidr_block" {
   type = "string"
 }

data/data/aws/vpc/vpc-private.tf

@@ -3,7 +3,7 @@ resource "aws_route_table" "private_routes" {
   vpc_id = "${data.aws_vpc.cluster_vpc.id}"
 
   tags = "${merge(map(
-    "Name","${var.cluster_id}-private-${local.new_subnet_azs[count.index]}",
+    "Name","${var.cluster_id}-private-${var.availability_zones[count.index]}",
   ), var.tags)}"
 }
 
@@ -22,10 +22,10 @@ resource "aws_subnet" "private_subnet" {
 
   cidr_block = "${cidrsubnet(local.new_private_cidr_range, 3, count.index)}"
 
-  availability_zone = "${local.new_subnet_azs[count.index]}"
+  availability_zone = "${var.availability_zones[count.index]}"
 
   tags = "${merge(map(
-    "Name", "${var.cluster_id}-private-${local.new_subnet_azs[count.index]}",
+    "Name", "${var.cluster_id}-private-${var.availability_zones[count.index]}",
     "kubernetes.io/role/internal-elb", "",
   ), var.tags)}"
 }

data/data/aws/vpc/vpc-public.tf

@@ -31,10 +31,10 @@ resource "aws_subnet" "public_subnet" {
 
   cidr_block = "${cidrsubnet(local.new_public_cidr_range, 3, count.index)}"
 
-  availability_zone = "${local.new_subnet_azs[count.index]}"
+  availability_zone = "${var.availability_zones[count.index]}"
 
   tags = "${merge(map(
-    "Name", "${var.cluster_id}-public-${local.new_subnet_azs[count.index]}",
+    "Name", "${var.cluster_id}-public-${var.availability_zones[count.index]}",
   ), var.tags)}"
 }
 
@@ -49,7 +49,7 @@ resource "aws_eip" "nat_eip" {
   vpc   = true
 
   tags = "${merge(map(
-    "Name", "${var.cluster_id}-eip-${local.new_subnet_azs[count.index]}",
+    "Name", "${var.cluster_id}-eip-${var.availability_zones[count.index]}",
   ), var.tags)}"
 
   # Terraform does not declare an explicit dependency towards the internet gateway.
@@ -64,6 +64,6 @@ resource "aws_nat_gateway" "nat_gw" {
   subnet_id     = "${aws_subnet.public_subnet.*.id[count.index]}"
 
   tags = "${merge(map(
-    "Name", "${var.cluster_id}-nat-${local.new_subnet_azs[count.index]}",
+    "Name", "${var.cluster_id}-nat-${var.availability_zones[count.index]}",
   ), var.tags)}"
 }

docs/user/aws/limits.md

@@ -23,25 +23,27 @@ limit.
 
 ## Elastic Network Interfaces (ENI)
 
-The default installation creates 21 + the number of availability zones of ENIs (e.g. us-east-1 = 21 + 6 = 27 ENIs).
+The default installation creates 21 + the number of availability zones of ENIs (e.g. 21 + 3 = 24 ENIs for a three-zone cluster).
 The default limit per region is 350. Additional ENIs are created for additional machines and elastic load balancers
 created by cluster usage and deployed workloads. A service limit increase here may be required to satisfy the needs of
 additional clusters and deployed workloads.
 
 ## Elastic IP (EIP)
 
-For a single, default cluster, your account will have the needed capacity limits required. There is one exception,
-"EC2-VPC Elastic IPs". The installer creates a public and private subnet for each
-[availability zone within a region][availability-zones] to provision the cluster in a highly available configuration. In
-each private subnet, a separate [NAT Gateway][nat-gateways] is created and requires a separate [elastic IP][elastic-ip].
-The default limit of 5 is sufficient for most regions and a single cluster. For the us-east-1 region, a higher limit is
-required. For multiple clusters, a higher limit is required. Please see [this map][az-map] for a current region map with
-availability zone count. We recommend selecting regions with 3 or more availability zones.
+By default, the installer distributes control-plane and compute machines across [all availability zones within a region][availability-zones] to provision the cluster in a highly available configuration.
+Please see [this map][az-map] for a current region map with availability zone count.
+We recommend selecting regions with 3 or more availability zones.
+You can [provide an install-config](../overview.md#multiple-invocations) to [configure](customization.md) the installer to use specific zones to override that default.
 
-### Example: Using N. Virginia (us-east-1)
+The installer creates a public and private subnet for each configured availability zone.
+In each private subnet, a separate [NAT Gateway][nat-gateways] is created and requires a separate [EC2-VPC Elastic IP (EIP)][elastic-ip].
+The default limit of 5 is sufficient for a single cluster, unless you have configured your cluster to use more than five zones.
+For multiple clusters, a higher limit will likely be required (and will certainly be required to support more than five clusters, even if they are each single-zone clusters).
 
-To use N. Virginia (us-east-1) for a new cluster, please submit a limit increase for VPC Elastic IPs similar to the
-following in the support dashboard (to create more than one cluster, a higher limit will be necessary):
+### Example: Using North Virginia (us-east-1)
+
+North Virginia (us-east-1) has six availablity zones, so a higher limit is required unless you configure your cluster to use fewer zones.
+To support the default, all-zone installation, please submit a limit increase for VPC Elastic IPs similar to the following in the support dashboard (to create more than one cluster, a higher limit will be necessary):
 
 ![Increase Elastic IP limit in AWS](images/support_increase_elastic_ip.png)
 

pkg/asset/cluster/tfvars.go

@@ -61,6 +61,7 @@ func (t *TerraformVariables) Dependencies() []asset.Asset {
 		&bootstrap.Bootstrap{},
 		&machine.Master{},
 		&machines.Master{},
+		&machines.Worker{},
 	}
 }
 
@@ -71,8 +72,9 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 	bootstrapIgnAsset := &bootstrap.Bootstrap{}
 	masterIgnAsset := &machine.Master{}
 	mastersAsset := &machines.Master{}
+	workersAsset := &machines.Worker{}
 	rhcosImage := new(rhcos.Image)
-	parents.Get(clusterID, installConfig, bootstrapIgnAsset, masterIgnAsset, mastersAsset, rhcosImage)
+	parents.Get(clusterID, installConfig, bootstrapIgnAsset, masterIgnAsset, mastersAsset, workersAsset, rhcosImage)
 
 	platform := installConfig.Config.Platform.Name()
 	switch platform {
@@ -83,8 +85,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 	bootstrapIgn := string(bootstrapIgnAsset.Files()[0].Data)
 	masterIgn := string(masterIgnAsset.Files()[0].Data)
 
-	masters := mastersAsset.Machines()
-	masterCount := len(masters)
+	masterCount := len(mastersAsset.MachineFiles)
 	data, err := tfvars.TFVars(
 		clusterID.InfraID,
 		installConfig.Config.ClusterDomain(),
@@ -110,15 +111,23 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 
 	switch platform {
 	case aws.Name:
-		masters, err := mastersAsset.StructuredMachines()
+		masters, err := mastersAsset.Machines()
 		if err != nil {
 			return err
 		}
 		masterConfigs := make([]*awsprovider.AWSMachineProviderConfig, len(masters))
 		for i, m := range masters {
 			masterConfigs[i] = m.Spec.ProviderSpec.Value.Object.(*awsprovider.AWSMachineProviderConfig)
 		}
-		data, err := awstfvars.TFVars(masterConfigs)
+		workers, err := workersAsset.MachineSets()
+		if err != nil {
+			return err
+		}
+		workerConfigs := make([]*awsprovider.AWSMachineProviderConfig, len(workers))
+		for i, m := range workers {
+			workerConfigs[i] = m.Spec.Template.Spec.ProviderSpec.Value.Object.(*awsprovider.AWSMachineProviderConfig)
+		}
+		data, err := awstfvars.TFVars(masterConfigs, workerConfigs)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)
 		}
@@ -127,7 +136,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			Data:     data,
 		})
 	case libvirt.Name:
-		masters, err := mastersAsset.StructuredMachines()
+		masters, err := mastersAsset.Machines()
 		if err != nil {
 			return err
 		}
@@ -146,7 +155,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			Data:     data,
 		})
 	case openstack.Name:
-		masters, err := mastersAsset.StructuredMachines()
+		masters, err := mastersAsset.Machines()
 		if err != nil {
 			return err
 		}

pkg/asset/ignition/bootstrap/bootstrap.go

@@ -57,6 +57,7 @@ func (a *Bootstrap) Dependencies() []asset.Asset {
 		&kubeconfig.AdminClient{},
 		&kubeconfig.Kubelet{},
 		&machines.Master{},
+		&machines.Worker{},
 		&manifests.Manifests{},
 		&manifests.Openshift{},
 		&tls.AdminKubeConfigCABundle{},
@@ -368,6 +369,7 @@ func (a *Bootstrap) addParentFiles(dependencies asset.Parents) {
 		&manifests.Manifests{},
 		&manifests.Openshift{},
 		&machines.Master{},
+		&machines.Worker{},
 	} {
 		dependencies.Get(asset)
 		a.Config.Storage.Files = append(a.Config.Storage.Files, ignition.FilesFromAsset(rootDir, "root", 0644, asset)...)

pkg/asset/machines/machineconfig/manifest.go

@@ -46,8 +46,12 @@ func Manifests(configs []*mcfgv1.MachineConfig, role, directory string) ([]*asse
 }
 
 // IsManifest tests whether the specified filename is a MachineConfig manifest.
-func IsManifest(role, filename string) bool {
-	return fmt.Sprintf(machineConfigFileName, role) == filename
+func IsManifest(filename string) (bool, error) {
+	matched, err := filepath.Match(machineConfigFileNamePattern, filename)
+	if err != nil {
+		return false, err
+	}
+	return matched, nil
 }
 
 // Load loads the MachineConfig manifests.

pkg/asset/machines/master.go

@@ -218,17 +218,8 @@ func (m *Master) Load(f asset.FileFetcher) (found bool, err error) {
 	return true, nil
 }
 
-// Machines returns master Machine manifest YAML.
-func (m *Master) Machines() [][]byte {
-	machines := make([][]byte, len(m.MachineFiles))
-	for i, file := range m.MachineFiles {
-		machines[i] = file.Data
-	}
-	return machines
-}
-
-// StructuredMachines returns master Machine manifest structures.
-func (m *Master) StructuredMachines() ([]machineapi.Machine, error) {
+// Machines returns master Machine manifest structures.
+func (m *Master) Machines() ([]machineapi.Machine, error) {
 	scheme := runtime.NewScheme()
 	awsapi.AddToScheme(scheme)
 	libvirtapi.AddToScheme(scheme)
@@ -259,21 +250,28 @@ func (m *Master) StructuredMachines() ([]machineapi.Machine, error) {
 	return machines, nil
 }
 
-// IsMasterManifest tests whether a file is a manifest that belongs to the
-// Master Machines asset.
-func IsMasterManifest(file *asset.File) bool {
+// IsMachineManifest tests whether a file is a manifest that belongs to the
+// Master Machines or Worker Machines asset.
+func IsMachineManifest(file *asset.File) bool {
 	if filepath.Dir(file.Filename) != directory {
 		return false
 	}
 	filename := filepath.Base(file.Filename)
-	if filename == masterUserDataFileName {
+	if filename == masterUserDataFileName || filename == workerUserDataFileName {
 		return true
 	}
-	if machineconfig.IsManifest("master", filename) {
+	if matched, err := machineconfig.IsManifest(filename); err != nil {
+		panic(err)
+	} else if matched {
 		return true
 	}
 	if matched, err := filepath.Match(masterMachineFileNamePattern, filename); err != nil {
 		panic("bad format for master machine file name pattern")
+	} else if matched {
+		return true
+	}
+	if matched, err := filepath.Match(workerMachineSetFileNamePattern, filename); err != nil {
+		panic("bad format for worker machine file name pattern")
 	} else {
 		return matched
 	}