diff --git a/Makefile b/Makefile index f12f389d93507..f6e5485086acb 100644 --- a/Makefile +++ b/Makefile @@ -198,6 +198,7 @@ test-e2e: --cloud-provider=aws \ --kops-binary-path=/home/prow/go/src/k8s.io/kops/bazel-bin/cmd/kops/linux-amd64/kops \ --kubernetes-version=v1.19.4 \ + --template-path=tests/e2e/templates/simple.yaml.tmpl \ --test=kops \ -- \ --test-package-version=v1.19.4 \ diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 95432c6ab221b..6e5d35bf372e1 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/octago/sflags v0.2.0 github.com/spf13/pflag v1.0.5 + gopkg.in/yaml.v2 v2.3.0 k8s.io/klog/v2 v2.4.0 sigs.k8s.io/kubetest2 v0.0.0-20201130212850-d9dad7c8699c ) diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index c729db658362c..58789226f2fcf 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -1493,6 +1493,7 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index bbcacb385b3af..9075d99bb679e 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -85,6 +85,10 @@ func (d *deployer) verifyKopsFlags() error { return errors.New("unsupported --cloud-provider value") } + if d.StateStore == "" { + d.StateStore = stateStore(d.CloudProvider) + } + return nil } @@ -94,7 +98,7 @@ func (d *deployer) env() []string { vars = append(vars, []string{ fmt.Sprintf("PATH=%v", os.Getenv("PATH")), fmt.Sprintf("HOME=%v", os.Getenv("HOME")), - fmt.Sprintf("KOPS_STATE_STORE=%v", stateStore(d.CloudProvider)), + fmt.Sprintf("KOPS_STATE_STORE=%v", d.StateStore), fmt.Sprintf("KOPS_FEATURE_FLAGS=%v", d.featureFlags()), "KOPS_RUN_TOO_NEW_VERSION=1", }...) @@ -148,9 +152,9 @@ func stateStore(cloudProvider string) string { if ss == "" { switch cloudProvider { case "aws": - ss = "s3://k8s-kops-prow/" + ss = "s3://k8s-kops-prow" case "gce": - ss = "gs://k8s-kops-gce/" + ss = "gs://k8s-kops-gce" } } return ss diff --git a/tests/e2e/kubetest2-kops/deployer/create.go b/tests/e2e/kubetest2-kops/deployer/create.go new file mode 100644 index 0000000000000..8107543c2c49a --- /dev/null +++ b/tests/e2e/kubetest2-kops/deployer/create.go @@ -0,0 +1,60 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "strings" + + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +// create performs a `kops create -f` followed by `kops update cluster --yes` +func (d *deployer) replace() error { + args := []string{ + d.KopsBinaryPath, "create", + "--filename", d.manifestPath, + "--name", d.ClusterName, + } + klog.Info(strings.Join(args, " ")) + + cmd := exec.Command(args[0], args[1:]...) + cmd.SetEnv(d.env()...) + + exec.InheritOutput(cmd) + err := cmd.Run() + if err != nil { + return err + } + + args = []string{ + d.KopsBinaryPath, "update", "cluster", "--yes", + "--admin", + "--name", d.ClusterName, + } + klog.Info(strings.Join(args, " ")) + + cmd = exec.Command(args[0], args[1:]...) + cmd.SetEnv(d.env()...) + + exec.InheritOutput(cmd) + err = cmd.Run() + if err != nil { + return err + } + return nil +} diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index e69b763cbf8a5..81c37915889a0 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -47,6 +47,8 @@ type deployer struct { KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"` StateStore string `flag:"-"` + TemplatePath string `flag:"template-path" desc:"The path to the manifest template used for cluster creation"` + KubernetesVersion string `flag:"kubernetes-version" desc:"The kubernetes version to use in the cluster"` SSHPrivateKeyPath string `flag:"ssh-private-key" desc:"The path to the private key used for SSH access to instances"` @@ -58,6 +60,9 @@ type deployer struct { AdminAccess string `flag:"admin-access" desc:"The CIDR to restrict kubernetes API access"` BuildOptions *builder.BuildOptions + + // manifestPath is the location of the rendered manifest based on TemplatePath + manifestPath string } // assert that New implements types.NewDeployer diff --git a/tests/e2e/kubetest2-kops/deployer/template.go b/tests/e2e/kubetest2-kops/deployer/template.go new file mode 100644 index 0000000000000..38dd9b92968c6 --- /dev/null +++ b/tests/e2e/kubetest2-kops/deployer/template.go @@ -0,0 +1,80 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "io/ioutil" + "path" + "strings" + + "gopkg.in/yaml.v2" + "k8s.io/klog/v2" + "sigs.k8s.io/kubetest2/pkg/exec" +) + +// renderTemplate will render the manifest template with the provided values, +// setting the deployer's manifestPath +func (d *deployer) renderTemplate(values map[string]interface{}) error { + dir, err := ioutil.TempDir("", "kops-template") + if err != nil { + return err + } + + valuesBytes, err := yaml.Marshal(values) + if err != nil { + return err + } + valuesPath := path.Join(dir, "values.yaml") + ioutil.WriteFile(valuesPath, valuesBytes, 0644) + + manifestPath := path.Join(dir, "manifest.yaml") + d.manifestPath = manifestPath + + args := []string{ + d.KopsBinaryPath, "toolbox", "template", + "--template", d.TemplatePath, + "--output", manifestPath, + "--values", valuesPath, + } + klog.Info(strings.Join(args, " ")) + + cmd := exec.Command(args[0], args[1:]...) + cmd.SetEnv(d.env()...) + + exec.InheritOutput(cmd) + err = cmd.Run() + if err != nil { + return err + } + return nil +} + +func (d *deployer) templateValues(zones []string, publicIP string) (map[string]interface{}, error) { + publicKey, err := ioutil.ReadFile(d.SSHPublicKeyPath) + if err != nil { + return nil, err + } + return map[string]interface{}{ + "cloudProvider": d.CloudProvider, + "clusterName": d.ClusterName, + "kubernetesVersion": d.KubernetesVersion, + "publicIP": publicIP, + "stateStore": d.StateStore, + "zones": zones, + "sshPublicKey": string(publicKey), + }, nil +} diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go index ac85ac703351c..7b5ad759b36bc 100644 --- a/tests/e2e/kubetest2-kops/deployer/up.go +++ b/tests/e2e/kubetest2-kops/deployer/up.go @@ -17,6 +17,7 @@ limitations under the License. package deployer import ( + "fmt" "os" osexec "os/exec" "strings" @@ -43,6 +44,41 @@ func (d *deployer) Up() error { adminAccess = publicIP } + zones, err := d.zones() + if err != nil { + return err + } + + if d.TemplatePath != "" { + values, err := d.templateValues(zones, adminAccess) + if err != nil { + return err + } + if err := d.renderTemplate(values); err != nil { + return err + } + if err := d.replace(); err != nil { + return err + } + } else { + err := d.createCluster(zones, adminAccess) + if err != nil { + return err + } + } + isUp, err := d.IsUp() + if err != nil { + return err + } else if isUp { + klog.V(1).Infof("cluster reported as up") + } else { + klog.Errorf("cluster reported as down") + } + return nil +} + +func (d *deployer) createCluster(zones []string, adminAccess string) error { + args := []string{ d.KopsBinaryPath, "create", "cluster", "--name", d.ClusterName, @@ -83,20 +119,10 @@ func (d *deployer) Up() error { cmd.SetEnv(d.env()...) exec.InheritOutput(cmd) - err = cmd.Run() - if err != nil { - return err - } - - isUp, err := d.IsUp() + err := cmd.Run() if err != nil { return err - } else if isUp { - klog.V(1).Infof("cluster reported as up") - } else { - klog.Errorf("cluster reported as down") } - return nil } @@ -133,3 +159,13 @@ func (d *deployer) verifyUpFlags() error { return nil } + +func (d *deployer) zones() ([]string, error) { + switch d.CloudProvider { + case "aws": + return aws.RandomZones(1) + case "gce": + return gce.RandomZones(1) + } + return nil, fmt.Errorf("unsupported CloudProvider: %v", d.CloudProvider) +} diff --git a/tests/e2e/templates/simple.yaml.tmpl b/tests/e2e/templates/simple.yaml.tmpl new file mode 100644 index 0000000000000..673f5b4eb8d32 --- /dev/null +++ b/tests/e2e/templates/simple.yaml.tmpl @@ -0,0 +1,90 @@ +{{$zone := index .zones 0}} +apiVersion: kops.k8s.io/v1alpha2 +kind: Cluster +metadata: + name: {{.clusterName}} +spec: + kubernetesApiAccess: + - {{.publicIP}} + channel: stable + cloudProvider: {{.cloudProvider}} + configBase: "{{.stateStore}}/{{.clusterName}}" + etcdClusters: + - etcdMembers: + - instanceGroup: master-{{$zone}} + name: {{$zone}} + name: main + - etcdMembers: + - instanceGroup: master-{{$zone}} + name: {{$zone}} + name: events + iam: {} + kubelet: + anonymousAuth: false + kubernetesVersion: {{.kubernetesVersion}} + masterInternalName: api.internal.{{.clusterName}} + masterPublicName: api.{{.clusterName}} + networkCIDR: 172.20.0.0/16 + networking: + kubenet: {} + nodePortAccess: + - 0.0.0.0/0 + nonMasqueradeCIDR: 100.64.0.0/10 + sshAccess: + - {{.publicIP}} + topology: + masters: public + nodes: public + subnets: + - cidr: 172.20.32.0/19 + name: {{$zone}} + type: Public + zone: {{$zone}} + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: SSHCredential +metadata: + name: admin + labels: + kops.k8s.io/cluster: {{.clusterName}} +spec: + publicKey: {{.sshPublicKey}} + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + name: nodes-{{$zone}} + labels: + kops.k8s.io/cluster: {{.clusterName}} +spec: + associatePublicIp: true + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201112.1 + machineType: t3.medium + maxSize: 4 + minSize: 4 + role: Node + subnets: + - {{$zone}} + +--- + +apiVersion: kops.k8s.io/v1alpha2 +kind: InstanceGroup +metadata: + name: master-{{$zone}} + labels: + kops.k8s.io/cluster: {{.clusterName}} +spec: + associatePublicIp: true + image: 099720109477/ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20201112.1 + machineType: c5.large + maxSize: 1 + minSize: 1 + role: Master + subnets: + - {{$zone}} +