Skip to content

Commit

Permalink
Merge pull request #4128 from cPu1/windows-private-cluster-fix
Browse files Browse the repository at this point in the history 
Fix creating Windows nodegroups in fully-private clusters
  • Loading branch information
@cPu1
cPu1 committed Aug 25, 2021
2 parents 0d3b889 + e8fcd98 commit 4a32940
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 118 deletions.
2 changes: 1 addition & 1 deletion pkg/nodebootstrap/userdata.go
View file
Expand Up @@ -50,7 +50,7 @@ func NewBootstrapper(clusterConfig *api.ClusterConfig, ng *api.NodeGroup) (Boots
ng.ClusterDNS = clusterDNS
}
if api.IsWindowsImage(ng.AMIFamily) {
return NewWindowsBootstrapper(clusterConfig.Metadata.Name, ng), nil
return NewWindowsBootstrapper(clusterConfig, ng), nil
}
switch ng.AMIFamily {
case api.NodeImageFamilyUbuntu2004, api.NodeImageFamilyUbuntu1804:
Expand Down
107 changes: 82 additions & 25 deletions pkg/nodebootstrap/windows.go
View file
Expand Up @@ -3,7 +3,6 @@ package nodebootstrap
import (
"encoding/base64"
"fmt"
"sort"
"strconv"
"strings"

Expand All @@ -13,47 +12,105 @@ import (
)

type Windows struct {
clusterName string
ng *api.NodeGroup
clusterConfig *api.ClusterConfig
ng *api.NodeGroup
}

func NewWindowsBootstrapper(clusterName string, ng *api.NodeGroup) *Windows {
type keyValue struct {
key string
value string
}

func NewWindowsBootstrapper(clusterConfig *api.ClusterConfig, ng *api.NodeGroup) *Windows {
return &Windows{
clusterName: clusterName,
ng: ng,
clusterConfig: clusterConfig,
ng: ng,
}
}

func (b *Windows) UserData() (string, error) {
bootstrapScript := `<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
`
for _, command := range b.ng.PreBootstrapCommands {
bootstrapScript += fmt.Sprintf("%s\n", command)
bootstrapCommands := []string{
`<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"`,
}

kubeletOptions := map[string]string{
"node-labels": formatLabels(b.ng.Labels),
"register-with-taints": utils.FormatTaints(b.ng.Taints),
bootstrapCommands = append(bootstrapCommands, b.ng.PreBootstrapCommands...)
eksBootstrapCommand := fmt.Sprintf("& $EKSBootstrapScriptFile %s 3>&1 4>&1 5>&1 6>&1", b.makeBootstrapParams())
bootstrapCommands = append(bootstrapCommands,
eksBootstrapCommand,
"</powershell>",
)

userData := base64.StdEncoding.EncodeToString([]byte(strings.Join(bootstrapCommands, "\n")))

logger.Debug("user-data = %s", userData)
return userData, nil
}

func (b *Windows) makeBootstrapParams() string {
params := []keyValue{
{
key: "EKSClusterName",
value: b.clusterConfig.Metadata.Name,
},
{
key: "APIServerEndpoint",
value: b.clusterConfig.Status.Endpoint,
},
{
key: "Base64ClusterCA",
value: base64.StdEncoding.EncodeToString(b.clusterConfig.Status.CertificateAuthorityData),
},
{
key: "DNSClusterIP",
value: b.ng.ClusterDNS,
},
{
key: "KubeletExtraArgs",
value: b.makeKubeletOptions(),
},
}
if b.ng.MaxPodsPerNode != 0 {
kubeletOptions["max-pods"] = strconv.Itoa(b.ng.MaxPodsPerNode)

return formatWindowsParams(params)
}

func (b *Windows) makeKubeletOptions() string {
kubeletOptions := []keyValue{
{
key: "node-labels",
value: formatLabels(b.ng.Labels),
},
{
key: "register-with-taints",
value: utils.FormatTaints(b.ng.Taints),
},
}

kubeletArgs := toCLIArgs(kubeletOptions)
bootstrapScript += fmt.Sprintf("& $EKSBootstrapScriptFile -EKSClusterName %q -KubeletExtraArgs %q 3>&1 4>&1 5>&1 6>&1\n</powershell>", b.clusterName, kubeletArgs)
if b.ng.MaxPodsPerNode != 0 {
kubeletOptions = append(kubeletOptions, keyValue{
key: "max-pods",
value: strconv.Itoa(b.ng.MaxPodsPerNode),
})
}

userData := base64.StdEncoding.EncodeToString([]byte(bootstrapScript))
return toCLIArgs(kubeletOptions)
}

logger.Debug("user-data = %s", userData)
return userData, nil
// formatWindowsParams formats params into `-key "value"`, ignoring keys with empty values
func formatWindowsParams(params []keyValue) string {
var args []string
for _, param := range params {
if param.value != "" {
args = append(args, fmt.Sprintf("-%s %q", param.key, param.value))
}
}
return strings.Join(args, " ")
}

func toCLIArgs(values map[string]string) string {
func toCLIArgs(params []keyValue) string {
var args []string
for k, v := range values {
args = append(args, fmt.Sprintf("--%s=%s", k, v))
for _, param := range params {
args = append(args, fmt.Sprintf("--%s=%s", param.key, param.value))
}
sort.Strings(args)
return strings.Join(args, " ")
}
185 changes: 93 additions & 92 deletions pkg/nodebootstrap/windows_test.go
View file
Expand Up @@ -5,134 +5,135 @@ import (
"strings"

. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/extensions/table"
. "github.com/onsi/gomega"

api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
"github.com/weaveworks/eksctl/pkg/nodebootstrap"
)

var _ = Describe("Windows", func() {
var (
clusterName string
ng *api.NodeGroup
)

BeforeEach(func() {
clusterName = "windohs"
ng = &api.NodeGroup{
type windowsEntry struct {
updateNodeGroup func(*api.NodeGroup)

expectedUserData string
}

DescribeTable("Windows bootstrap", func(e windowsEntry) {
clusterConfig := api.NewClusterConfig()
clusterConfig.Metadata.Name = "windohs"
clusterConfig.Status = &api.ClusterStatus{
Endpoint: "https://test.com",
CertificateAuthorityData: []byte("test"),
}
ng := &api.NodeGroup{
NodeGroupBase: &api.NodeGroupBase{
AMIFamily: api.NodeImageFamilyWindowsServer2019CoreContainer,
},
}
})
if e.updateNodeGroup != nil {
e.updateNodeGroup(ng)
}

It("produces correct standard userdata", func() {
ng.PreBootstrapCommands = nil
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
bootstrapper := nodebootstrap.NewWindowsBootstrapper(clusterConfig, ng)
userData, err := bootstrapper.UserData()
Expect(err).ToNot(HaveOccurred())

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
Expect(decodeData(userData)).To(Equal(strings.TrimSpace(e.expectedUserData)))
},
Entry("standard userdata", windowsEntry{

expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})

When("labels are set on the node", func() {
It("adds them to the userdata", func() {
ng.Labels = map[string]string{"foo": "bar"}
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
Expect(err).ToNot(HaveOccurred())
`,
}),

Entry("with labels", windowsEntry{
updateNodeGroup: func(ng *api.NodeGroup) {
ng.Labels = map[string]string{
"foo": "bar",
}
},

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--node-labels=foo=bar --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels=foo=bar --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})
})

When("taints are set on the node", func() {
It("adds them to the userdata", func() {
ng.Taints = []api.NodeGroupTaint{
{
Key: "foo",
Value: "bar",
Effect: "NoSchedule",
},
}
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
Expect(err).ToNot(HaveOccurred())

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
`,
}),

Entry("with taints", windowsEntry{
updateNodeGroup: func(ng *api.NodeGroup) {
ng.Taints = []api.NodeGroupTaint{
{
Key: "foo",
Value: "bar",
Effect: "NoSchedule",
},
}
},

expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--node-labels= --register-with-taints=foo=bar:NoSchedule" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels= --register-with-taints=foo=bar:NoSchedule" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})
})

When("MaxPodsPerNode are set", func() {
It("adds them to the userdata", func() {
ng.MaxPodsPerNode = 100
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
Expect(err).ToNot(HaveOccurred())

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
`,
}),

Entry("with maxPods", windowsEntry{
updateNodeGroup: func(ng *api.NodeGroup) {
ng.MaxPodsPerNode = 100
},

expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--max-pods=100 --node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels= --register-with-taints= --max-pods=100" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})
})

When("a PreBootstrapCommands is set", func() {
It("adds it to the userdata", func() {
ng.PreBootstrapCommands = []string{
"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
}
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
Expect(err).ToNot(HaveOccurred())

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
`,
}),

Entry("with a preBootstrapCommand", windowsEntry{
updateNodeGroup: func(ng *api.NodeGroup) {
ng.PreBootstrapCommands = []string{
"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
}
},

expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})
})

When("several PreBootstrapCommands are set", func() {
It("adds them to the userdata", func() {
ng.PreBootstrapCommands = []string{
"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
"start /wait msiexec.exe /qb /i \"amazon-cloudwatch-agent.msi\"",
}
bootstrap := nodebootstrap.NewWindowsBootstrapper(clusterName, ng)
userdata, err := bootstrap.UserData()
Expect(err).ToNot(HaveOccurred())

Expect(decodeData(userdata)).To(Equal(strings.TrimSpace(`
`,
}),

Entry("with several preBootstrapCommands", windowsEntry{
updateNodeGroup: func(ng *api.NodeGroup) {
ng.PreBootstrapCommands = []string{
"wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi",
"start /wait msiexec.exe /qb /i \"amazon-cloudwatch-agent.msi\"",
}

},

expectedUserData: `
<powershell>
[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
wget -UseBasicParsing -O amazon-cloudwatch-agent.msi https://s3.amazonaws.com/amazoncloudwatch-agent/windows/amd64/latest/amazon-cloudwatch-agent.msi
start /wait msiexec.exe /qb /i "amazon-cloudwatch-agent.msi"
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
& $EKSBootstrapScriptFile -EKSClusterName "windohs" -APIServerEndpoint "https://test.com" -Base64ClusterCA "dGVzdA==" -KubeletExtraArgs "--node-labels= --register-with-taints=" 3>&1 4>&1 5>&1 6>&1
</powershell>
`)))
})
})
`,
}),
)

})

func decodeData(userdata string) string {
Expand Down

0 comments on commit 4a32940

Please sign in to comment.