Add Services Support for EKS and EMR (#72)

* support service aggregate filters for eks and emr

* add eks tests

* add virt type test

* add service tests

* add emr tests

* address pr comments

* add semver license

Author: bwagner5

THIRD_PARTY_LICENSES

@@ -784,3 +784,63 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
+------
+
+** github.com/imdario/mergo; version v0.3.11 --
+https://github.com/imdario/mergo
+
+Copyright (c) 2013 Dario Castañé. All rights reserved.
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+------
+
+** github.com/blang/semver; version v4.0.0 --
+https://github.com/blang/semver
+
+The MIT License
+
+Copyright (c) 2014 Benedikt Lang <github at benediktlang.de>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

cmd/main.go

@@ -68,12 +68,14 @@ const (
 	networkPerformance     = "network-performance"
 	allowList              = "allow-list"
 	denyList               = "deny-list"
+	virtualizationType     = "virtualization-type"
 )
 
 // Aggregate Filter Flags
 const (
 	instanceTypeBase = "base-instance-type"
 	flexible         = "flexible"
+	service          = "service"
 )
 
 // Configuration Flag Constants
@@ -121,29 +123,31 @@ Full docs can be found at github.com/aws/amazon-` + binName
 	cli.IntMinMaxRangeFlags(vcpus, cli.StringMe("c"), nil, "Number of vcpus available to the instance type.")
 	cli.ByteQuantityMinMaxRangeFlags(memory, cli.StringMe("m"), nil, "Amount of Memory available (Example: 4 GiB)")
 	cli.RatioFlag(vcpusToMemoryRatio, nil, nil, "The ratio of vcpus to GiBs of memory. (Example: 1:2)")
-	cli.StringFlag(cpuArchitecture, cli.StringMe("a"), nil, "CPU architecture [x86_64/amd64, i386, or arm64]", nil)
+	cli.StringOptionsFlag(cpuArchitecture, cli.StringMe("a"), nil, "CPU architecture [x86_64/amd64, i386, or arm64]", []string{"x86_64", "amd64", "i386", "arm64"})
 	cli.IntMinMaxRangeFlags(gpus, cli.StringMe("g"), nil, "Total Number of GPUs (Example: 4)")
 	cli.ByteQuantityMinMaxRangeFlags(gpuMemoryTotal, nil, nil, "Number of GPUs' total memory (Example: 4 GiB)")
-	cli.StringFlag(placementGroupStrategy, nil, nil, "Placement group strategy: [cluster, partition, spread]", nil)
-	cli.StringFlag(usageClass, cli.StringMe("u"), nil, "Usage class: [spot or on-demand]", nil)
-	cli.StringFlag(rootDeviceType, nil, nil, "Supported root device types: [ebs or instance-store]", nil)
+	cli.StringOptionsFlag(placementGroupStrategy, nil, nil, "Placement group strategy: [cluster, partition, spread]", []string{"cluster", "partition", "spread"})
+	cli.StringOptionsFlag(usageClass, cli.StringMe("u"), nil, "Usage class: [spot or on-demand]", []string{"spot", "on-demand"})
+	cli.StringOptionsFlag(rootDeviceType, nil, nil, "Supported root device types: [ebs or instance-store]", []string{"ebs", "instance-store"})
 	cli.BoolFlag(enaSupport, cli.StringMe("e"), nil, "Instance types where ENA is supported or required")
 	cli.BoolFlag(hibernationSupport, nil, nil, "Hibernation supported")
 	cli.BoolFlag(baremetal, nil, nil, "Bare Metal instance types (.metal instances)")
 	cli.BoolFlag(fpgaSupport, cli.StringMe("f"), nil, "FPGA instance types")
 	cli.BoolFlag(burstSupport, cli.StringMe("b"), nil, "Burstable instance types")
-	cli.StringFlag(hypervisor, nil, nil, "Hypervisor: [xen or nitro]", nil)
+	cli.StringOptionsFlag(hypervisor, nil, nil, "Hypervisor: [xen or nitro]", []string{"xen", "nitro"})
 	cli.StringSliceFlag(availabilityZones, cli.StringMe("z"), nil, "Availability zones or zone ids to check EC2 capacity offered in specific AZs")
 	cli.BoolFlag(currentGeneration, nil, nil, "Current generation instance types (explicitly set this to false to not return current generation instance types)")
 	cli.IntMinMaxRangeFlags(networkInterfaces, nil, nil, "Number of network interfaces (ENIs) that can be attached to the instance")
 	cli.IntMinMaxRangeFlags(networkPerformance, nil, nil, "Bandwidth in Gib/s of network performance (Example: 100)")
 	cli.RegexFlag(allowList, nil, nil, "List of allowed instance types to select from w/ regex syntax (Example: m[3-5]\\.*)")
 	cli.RegexFlag(denyList, nil, nil, "List of instance types which should be excluded w/ regex syntax (Example: m[1-2]\\.*)")
+	cli.StringOptionsFlag(virtualizationType, nil, nil, "Virtualization Type supported: [hvm or pv]", []string{"hvm", "paravirtual", "pv"})
 
 	// Suite Flags - higher level aggregate filters that return opinionated result
 
 	cli.SuiteStringFlag(instanceTypeBase, nil, nil, "Instance Type used to retrieve similarly spec'd instance types", nil)
 	cli.SuiteBoolFlag(flexible, nil, nil, "Retrieves a group of instance types spanning multiple generations based on opinionated defaults and user overridden resource filters")
+	cli.SuiteStringFlag(service, nil, nil, "Filter instance types based on service support (Example: eks, eks-20201211, or emr-5.20.0)", nil)
 
 	// Configuration Flags - These will be grouped at the bottom of the help flags
 
@@ -206,6 +210,8 @@ Full docs can be found at github.com/aws/amazon-` + binName
 		DenyList:               cli.RegexMe(flags[denyList]),
 		InstanceTypeBase:       cli.StringMe(flags[instanceTypeBase]),
 		Flexible:               cli.BoolMe(flags[flexible]),
+		Service:                cli.StringMe(flags[service]),
+		VirtualizationType:     cli.StringMe(flags[virtualizationType]),
 	}
 
 	if flags[verbose] != nil {

go.mod

@@ -4,8 +4,10 @@ go 1.15
 
 require (
 	github.com/aws/aws-sdk-go v1.31.12
+	github.com/blang/semver/v4 v4.0.0
 	github.com/ghodss/yaml v1.0.0
 	github.com/hashicorp/hcl v1.0.0
+	github.com/imdario/mergo v0.3.11
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/cobra v0.0.7

go.sum

@@ -8,6 +8,8 @@ github.com/aws/aws-sdk-go v1.31.12 h1:SxRRGyhlCagI0DYkhOg+FgdXGXzRTE3vEX/gsgFaiK
 github.com/aws/aws-sdk-go v1.31.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -49,6 +51,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
@@ -166,4 +170,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pkg/selector/aggregates.go

@@ -48,7 +48,7 @@ func (itf Selector) TransformBaseInstanceType(filters Filters) (Filters, error)
 	if filters.BareMetal == nil {
 		filters.BareMetal = instanceTypeInfo.BareMetal
 	}
-	if filters.CPUArchitecture == nil {
+	if filters.CPUArchitecture == nil && len(instanceTypeInfo.ProcessorInfo.SupportedArchitectures) == 1 {
 		filters.CPUArchitecture = instanceTypeInfo.ProcessorInfo.SupportedArchitectures[0]
 	}
 	if filters.Fpga == nil {
@@ -72,6 +72,9 @@ func (itf Selector) TransformBaseInstanceType(filters Filters) (Filters, error)
 		upperBound := int(float64(*instanceTypeInfo.VCpuInfo.DefaultVCpus) * AggregateHighPercentile)
 		filters.VCpusRange = &IntRangeFilter{LowerBound: lowerBound, UpperBound: upperBound}
 	}
+	if filters.VirtualizationType == nil && len(instanceTypeInfo.SupportedVirtualizationTypes) == 1 {
+		filters.VirtualizationType = instanceTypeInfo.SupportedVirtualizationTypes[0]
+	}
 	filters.InstanceTypeBase = nil
 
 	return filters, nil
@@ -107,3 +110,8 @@ func (itf Selector) TransformFlexible(filters Filters) (Filters, error) {
 
 	return filters, nil
 }
+
+// TransformForService transforms lower level filters based on the service
+func (itf Selector) TransformForService(filters Filters) (Filters, error) {
+	return itf.ServiceRegistry.ExecuteTransforms(filters)
+}

pkg/selector/eks.go

@@ -0,0 +1,137 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package selector
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+const (
+	eksAMIRepoURL               = "https://github.com/awslabs/amazon-eks-ami"
+	eksFallbackLatestAMIVersion = "v20210125"
+	eksInstanceTypesFile        = "eni-max-pods.txt"
+)
+
+// EKS is a Service type for a custom service filter transform
+type EKS struct {
+	AMIRepoURL string
+}
+
+// Filters implements the Service interface contract for EKS
+func (e *EKS) Filters(version string) (Filters, error) {
+	if e.AMIRepoURL == "" {
+		e.AMIRepoURL = eksAMIRepoURL
+	}
+	filters := Filters{}
+
+	if version == "" {
+		var err error
+		version, err = e.getLatestAMIVersion()
+		if err != nil {
+			log.Printf("There was a problem fetching the latest EKS AMI version, using hardcoded fallback version %s\n", eksFallbackLatestAMIVersion)
+			version = eksFallbackLatestAMIVersion
+		}
+	}
+	if !strings.HasPrefix(version, "v") {
+		version = fmt.Sprintf("v%s", version)
+	}
+	supportedInstanceTypes, err := e.getSupportedInstanceTypes(version)
+	if err != nil {
+		log.Printf("Unable to retrieve EKS supported instance types for version %s: %v", version, err)
+		return filters, err
+	}
+	filters.InstanceTypes = &supportedInstanceTypes
+	filters.VirtualizationType = aws.String("hvm")
+	return filters, nil
+}
+
+func (e *EKS) getSupportedInstanceTypes(version string) ([]string, error) {
+	supportedInstanceTypes := []string{}
+	resp, err := http.Get(fmt.Sprintf("%s/archive/%s.zip", e.AMIRepoURL, version))
+	if err != nil {
+		return supportedInstanceTypes, err
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return supportedInstanceTypes, fmt.Errorf("Unable to retrieve EKS supported instance types, got non-200 status code: %d", resp.StatusCode)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return supportedInstanceTypes, err
+	}
+
+	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
+	if err != nil {
+		return supportedInstanceTypes, err
+	}
+
+	// Read all the files from zip archive
+	for _, zipFile := range zipReader.File {
+		filePathParts := strings.Split(zipFile.Name, "/")
+		fileName := filePathParts[len(filePathParts)-1]
+		if fileName == eksInstanceTypesFile {
+			unzippedFileBytes, err := readZipFile(zipFile)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			supportedInstanceTypesFileBody := string(unzippedFileBytes)
+			for _, line := range strings.Split(strings.Replace(supportedInstanceTypesFileBody, "\r\n", "\n", -1), "\n") {
+				if !strings.HasPrefix(line, "#") {
+					instanceType := strings.Split(line, " ")[0]
+					supportedInstanceTypes = append(supportedInstanceTypes, instanceType)
+				}
+			}
+		}
+	}
+	return supportedInstanceTypes, nil
+}
+
+func (e EKS) getLatestAMIVersion() (string, error) {
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	// Get latest version
+	resp, err := client.Get(fmt.Sprintf("%s/releases/latest", e.AMIRepoURL))
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != 302 {
+		return "", fmt.Errorf("Can't retrieve latest release from github because redirect was not sent")
+	}
+	versionRedirect := resp.Header.Get("location")
+	pathParts := strings.Split(versionRedirect, "/")
+	return pathParts[len(pathParts)-1], nil
+}
+
+func readZipFile(zf *zip.File) ([]byte, error) {
+	f, err := zf.Open()
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ioutil.ReadAll(f)
+}