Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test: Replace managed etcd test with generic etcd test #11544

Merged
merged 2 commits into from May 15, 2020
Merged

test: Replace managed etcd test with generic etcd test #11544

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
abe5ae0
test: Replace managed etcd test with etcd test
tgraf May 15, 2020
b3431dc
test: Remove leftover etcd operator related code
tgraf May 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
16 changes: 0 additions & 16 deletions test/helpers/cons.go
View file
Open in desktop
Expand Up @@ -265,16 +265,6 @@ var (
IsCiliumV1_8 = versioncheck.MustCompile(">=1.7.90 <1.9.0")
)

// CiliumDefaultDSPatch is the default Cilium DaemonSet patch to be used in all tests.
const CiliumDefaultDSPatch = "cilium-ds-patch.yaml"

// CiliumConfigMapPatch is the default Cilium ConfigMap patch to be used in all tests.
const CiliumConfigMapPatch = "cilium-cm-patch.yaml"

// CiliumConfigMapPatchKvstoreAllocator is equivalent to CiliumConfigMapPatch
// except it uses the kvstore-based allocator instead of the CRD-based allocator.
const CiliumConfigMapPatchKvstoreAllocator = "cilium-cm-kvstore-allocator-patch.yaml"

// badLogMessages is a map which key is a part of a log message which indicates
// a failure if the message does not contain any part from value list.
var badLogMessages = map[string][]string{
Expand Down Expand Up @@ -327,12 +317,6 @@ var ciliumKubCLICommandsKVStore = map[string]string{
"cilium kvstore get cilium --recursive": "kvstore_get.txt",
}

const (
ciliumEtcdOperatorSA = "cilium-etcd-operator-sa.yaml"
ciliumEtcdOperatorRBAC = "cilium-etcd-operator-rbac.yaml"
ciliumEtcdOperator = "cilium-etcd-operator.yaml"
)

// K8s1VMName is the name of the Kubernetes master node when running K8s tests.
func K8s1VMName() string {
return fmt.Sprintf("k8s1-%s", GetCurrentK8SEnv())
Expand Down
207 changes: 0 additions & 207 deletions test/helpers/kubectl.go
View file
Open in desktop
Expand Up @@ -92,7 +92,6 @@ var (
"preflight.image": "cilium-dev", // Set again in init to match agent.image!
"global.tag": "latest",
"operator.image": "operator",
"managed-etcd.registry": "docker.io/cilium",
"global.debug.enabled": "true",
"global.k8s.requireIPv4PodCIDR": "true",
"global.pprof.enabled": "true",
Expand Down Expand Up @@ -1553,94 +1552,6 @@ func (kub *Kubectl) DeployPatch(original, patchFileName string) error {
return nil
}

// ciliumInstall installs all Cilium descriptors into kubernetes.
// dsPatchName corresponds to the DaemonSet patch, found by
// getK8sDescriptorPatch, that will be applied to the original Cilium DaemonSet
// descriptor, found by getK8sDescriptor.
// cmPatchName corresponds to the ConfigMap patch, found by
// getK8sDescriptorPatch, that will be applied to the original Cilium ConfigMap
// descriptor, found by getK8sDescriptor.
// Returns an error if any patch or if any original descriptors files were not
// found.
func (kub *Kubectl) ciliumInstall(ciliumFilename, dsPatchName, cmPatchName string, getK8sDescriptor, getK8sDescriptorPatch func(filename string) string) error {
cmPathname := getK8sDescriptor("cilium-cm.yaml")
if cmPathname == "" {
return fmt.Errorf("Cilium ConfigMap descriptor not found")
}
dsPathname := getK8sDescriptor("cilium-ds.yaml")
if dsPathname == "" {
return fmt.Errorf("Cilium DaemonSet descriptor not found")
}
rbacPathname := getK8sDescriptor("cilium-rbac.yaml")
if rbacPathname == "" {
return fmt.Errorf("Cilium RBAC descriptor not found")
}

deployOriginal := func(original string) error {
// debugYaml only dumps the full created yaml file to the test output if
// the cilium manifest can not be created correctly.
debugYaml := func(original string) {
kub.Apply(ApplyOptions{
FilePath: original,
DryRun: true,
Output: "yaml",
})
}

// validation 1st
res := kub.Apply(ApplyOptions{
FilePath: original,
DryRun: true,
})
if !res.WasSuccessful() {
debugYaml(original)
return res.GetErr("Cilium manifest validation fails")
}

res = kub.ApplyDefault(original)
if !res.WasSuccessful() {
debugYaml(original)
return res.GetErr("Cannot apply Cilium manifest")
}
return nil
}

if err := deployOriginal(rbacPathname); err != nil {
return err
}

if err := kub.DeployPatch(cmPathname, getK8sDescriptorPatch(cmPatchName)); err != nil {
return err
}

if err := kub.DeployPatch(dsPathname, getK8sDescriptorPatch(dsPatchName)); err != nil {
return err
}

cmdRes := kub.ApplyDefault(getK8sDescriptor(ciliumEtcdOperatorSA))
if !cmdRes.WasSuccessful() {
return fmt.Errorf("Unable to deploy descriptor of etcd-operator SA %s: %s", ciliumEtcdOperatorSA, cmdRes.OutputPrettyPrint())
}

cmdRes = kub.ApplyDefault(getK8sDescriptor(ciliumEtcdOperatorRBAC))
if !cmdRes.WasSuccessful() {
return fmt.Errorf("Unable to deploy descriptor of etcd-operator RBAC %s: %s", ciliumEtcdOperatorRBAC, cmdRes.OutputPrettyPrint())
}

cmdRes = kub.ApplyDefault(getK8sDescriptor(ciliumEtcdOperator))
if !cmdRes.WasSuccessful() {
return fmt.Errorf("Unable to deploy descriptor of etcd-operator %s: %s", ciliumEtcdOperator, cmdRes.OutputPrettyPrint())
}

_ = kub.ApplyDefault(getK8sDescriptor("cilium-operator-sa.yaml"))
err := kub.DeployPatch(getK8sDescriptor("cilium-operator.yaml"), getK8sDescriptorPatch("cilium-operator-patch.yaml"))
if err != nil {
return fmt.Errorf("Unable to deploy descriptor of cilium-operators: %s", err)
}

return nil
}

func addIfNotOverwritten(options map[string]string, field, value string) map[string]string {
if _, ok := options[field]; !ok {
options[field] = value
Expand Down Expand Up @@ -1861,54 +1772,6 @@ func (kub *Kubectl) CiliumUninstall(filename string, options map[string]string)
return kub.ciliumUninstallHelm(filename, options)
}

// CiliumInstallVersion installs all Cilium descriptors into kubernetes for
// a given Cilium Version tag.
// dsPatchName corresponds to the DaemonSet patch that will be applied to the
// original Cilium DaemonSet descriptor of that given Cilium Version tag.
// cmPatchName corresponds to the ConfigMap patch that will be applied to the
// original Cilium ConfigMap descriptor of that given Cilium Version tag.
// Returns an error if any patch or if any original descriptors files were not
// found.
func (kub *Kubectl) CiliumInstallVersion(ciliumFilename, dsPatchName, cmPatchName, versionTag string) error {
getK8sDescriptorPatch := func(filename string) string {
// try dependent Cilium, k8s and integration version patch file
ginkgoVersionedPath := filepath.Join(manifestsPath, versionTag, GetCurrentK8SEnv(), GetCurrentIntegration(), filename)
_, err := os.Stat(ginkgoVersionedPath)
if err == nil {
return filepath.Join(kub.BasePath(), ginkgoVersionedPath)
}
// try dependent Cilium version and integration patch file
ginkgoVersionedPath = filepath.Join(manifestsPath, versionTag, GetCurrentIntegration(), filename)
_, err = os.Stat(ginkgoVersionedPath)
if err == nil {
return filepath.Join(kub.BasePath(), ginkgoVersionedPath)
}
// try dependent Cilium and k8s version patch file
ginkgoVersionedPath = filepath.Join(manifestsPath, versionTag, GetCurrentK8SEnv(), filename)
_, err = os.Stat(ginkgoVersionedPath)
if err == nil {
return filepath.Join(kub.BasePath(), ginkgoVersionedPath)
}
// try dependent Cilium version patch file
ginkgoVersionedPath = filepath.Join(manifestsPath, versionTag, filename)
_, err = os.Stat(ginkgoVersionedPath)
if err == nil {
return filepath.Join(kub.BasePath(), ginkgoVersionedPath)
}
// try dependent integration patch file
ginkgoVersionedPath = filepath.Join(manifestsPath, GetCurrentIntegration(), filename)
_, err = os.Stat(ginkgoVersionedPath)
if err == nil {
return filepath.Join(kub.BasePath(), ginkgoVersionedPath)
}
return filepath.Join(kub.BasePath(), manifestsPath, filename)
}
getK8sDescriptor := func(filename string) string {
return fmt.Sprintf("https://raw.githubusercontent.com/cilium/cilium/%s/examples/kubernetes/%s/%s", versionTag, GetCurrentK8SEnv(), filename)
}
return kub.ciliumInstall(ciliumFilename, dsPatchName, cmPatchName, getK8sDescriptor, getK8sDescriptorPatch)
}

// GetCiliumPods returns a list of all Cilium pods in the specified namespace,
// and an error if the Cilium pods were not able to be retrieved.
func (kub *Kubectl) GetCiliumPods(namespace string) ([]string, error) {
Expand Down Expand Up @@ -2482,40 +2345,6 @@ func (kub *Kubectl) CiliumReport(namespace string, commands ...string) {
}
}

// EtcdOperatorReport dump etcd pods data into the report directory to be able
// to debug etcd operator status in case of fail test.
func (kub *Kubectl) EtcdOperatorReport(ctx context.Context, reportCmds map[string]string) {
if reportCmds == nil {
reportCmds = make(map[string]string)
}

pods, err := kub.GetPodNamesContext(ctx, GetCiliumNamespace(GetCurrentIntegration()), "etcd_cluster=cilium-etcd")
if err != nil {
kub.Logger().WithError(err).Error("No etcd pods")
return
}

etcdctl := "etcdctl --endpoints=https://%s.cilium-etcd.%s.svc:2379 " +
"--cert-file /etc/etcdtls/member/peer-tls/peer.crt " +
"--key-file /etc/etcdtls/member/peer-tls/peer.key " +
"--ca-file /etc/etcdtls/member/peer-tls/peer-ca.crt " +
" %s"

etcdDumpCommands := map[string]string{
"member list": "etcd_%s_member_list",
"cluster-health": "etcd_%s_cluster_health",
}

for _, pod := range pods {
for cmd, reportFile := range etcdDumpCommands {
etcdCmd := fmt.Sprintf(etcdctl, pod, CiliumNamespace, cmd)
command := fmt.Sprintf("%s -n %s exec -ti %s -- %s",
KubectlCmd, GetCiliumNamespace(GetCurrentIntegration()), pod, etcdCmd)
reportCmds[command] = fmt.Sprintf(reportFile, pod)
}
}
}

// CiliumCheckReport prints a few checks on the Junit output to provide more
// context to users. The list of checks that prints are the following:
// - Number of Kubernetes and Cilium policies installed.
Expand Down Expand Up @@ -2868,7 +2697,6 @@ func (kub *Kubectl) GatherLogs(ctx context.Context) {
}

kub.GeneratePodLogGatheringCommands(ctx, reportCmds)
kub.EtcdOperatorReport(ctx, reportCmds)

res := kub.ExecContext(ctx, fmt.Sprintf(`%s api-resources | grep -v "^NAME" | awk '{print $1}'`, KubectlCmd))
if res.WasSuccessful() {
Expand Down Expand Up @@ -3399,41 +3227,6 @@ func (kub *Kubectl) ciliumServicePreFlightCheck() error {
return nil
}

// DeleteETCDOperator delete the etcd-operator from the cluster pointed by kub.
func (kub *Kubectl) DeleteETCDOperator() {
if res := kub.ExecShort(fmt.Sprintf("%s -n %s delete crd etcdclusters.etcd.database.coreos.com", KubectlCmd, GetCiliumNamespace(GetCurrentIntegration()))); !res.WasSuccessful() {
log.Warningf("Unable to delete etcdclusters.etcd.database.coreos.com CRD: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s -n %s delete deployment cilium-etcd-operator", KubectlCmd, GetCiliumNamespace(GetCurrentIntegration()))); !res.WasSuccessful() {
log.Warningf("Unable to delete cilium-etcd-operator Deployment: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s delete clusterrolebinding cilium-etcd-operator", KubectlCmd)); !res.WasSuccessful() {
log.Warningf("Unable to delete cilium-etcd-operator ClusterRoleBinding: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s delete clusterrole cilium-etcd-operator", KubectlCmd)); !res.WasSuccessful() {
log.Warningf("Unable to delete cilium-etcd-operator ClusterRole: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s -n %s delete serviceaccount cilium-etcd-operator", KubectlCmd, GetCiliumNamespace(GetCurrentIntegration()))); !res.WasSuccessful() {
log.Warningf("Unable to delete cilium-etcd-operator ServiceAccount: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s delete clusterrolebinding etcd-operator", KubectlCmd)); !res.WasSuccessful() {
log.Warningf("Unable to delete etcd-operator ClusterRoleBinding: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s delete clusterrole etcd-operator", KubectlCmd)); !res.WasSuccessful() {
log.Warningf("Unable to delete etcd-operator ClusterRole: %s", res.OutputPrettyPrint())
}

if res := kub.ExecShort(fmt.Sprintf("%s -n %s delete serviceaccount cilium-etcd-sa", KubectlCmd, GetCiliumNamespace(GetCurrentIntegration()))); !res.WasSuccessful() {
log.Warningf("Unable to delete cilium-etcd-sa ServiceAccount: %s", res.OutputPrettyPrint())
}
}

// reportMapContext saves the output of the given commands to the specified filename.
// Function needs a directory path where the files are going to be written
// commands are run on all pods matching selector
Expand Down
20 changes: 12 additions & 8 deletions test/k8sT/DatapathConfiguration.go
View file
Open in desktop
Expand Up @@ -62,7 +62,7 @@ var _ = Describe("K8sDatapathConfig", func() {

AfterFailed(func() {
kubectl.CiliumReport(helpers.CiliumNamespace,
"cilium bpf tunnel list",
"cilium status",
"cilium endpoint list")
})

Expand Down Expand Up @@ -580,14 +580,18 @@ var _ = Describe("K8sDatapathConfig", func() {
})
})

Context("ManagedEtcd", func() {
AfterAll(func() {
deleteETCDOperator(kubectl)
})
It("Check connectivity with managed etcd", func() {
Context("Etcd", func() {
It("Check connectivity", func() {
deploymentManager.Deploy(helpers.CiliumNamespace, StatelessEtcd)
deploymentManager.WaitUntilReady()

aanm marked this conversation as resolved.
Show resolved Hide resolved
host, port, err := kubectl.GetServiceHostPort(helpers.CiliumNamespace, "stateless-etcd")
Expect(err).Should(BeNil(), "Unable to retrieve ClusterIP and port for stateless-etcd service")

etcdService := fmt.Sprintf("http://%s:%d", host, port)
opts := map[string]string{
"global.etcd.enabled": "true",
"global.etcd.managed": "true",
"global.etcd.enabled": "true",
"global.etcd.endpoints[0]": etcdService,
}
if helpers.ExistNodeWithoutCilium() {
opts["global.synchronizeK8sNodes"] = "false"
Expand Down
2 changes: 0 additions & 2 deletions test/k8sT/Nightly.go
View file
Open in desktop
Expand Up @@ -395,8 +395,6 @@ var _ = Describe("NightlyExamples", func() {
kubectl.Exec(fmt.Sprintf(
"%s delete --all pods,svc,cnp -n %s", helpers.KubectlCmd, helpers.DefaultNamespace))

kubectl.DeleteETCDOperator()

ExpectAllPodsTerminated(kubectl)
})

Expand Down
11 changes: 1 addition & 10 deletions test/k8sT/Updates.go
View file
Open in desktop
Expand Up @@ -82,8 +82,6 @@ var _ = Describe("K8sUpdates", func() {
kubectl.Exec(fmt.Sprintf(
"%s delete --all pods,svc,cnp -n %s", helpers.KubectlCmd, helpers.DefaultNamespace))

kubectl.DeleteETCDOperator()

ExpectAllPodsTerminated(kubectl)
})

Expand Down Expand Up @@ -189,8 +187,6 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers
kubectl.Delete(l7Policy)
kubectl.Delete(demoPath)

kubectl.DeleteETCDOperator()

if res := kubectl.DeleteResource("pod", fmt.Sprintf("-n %s -l k8s-app=kube-dns", helpers.KubeSystemNamespace)); !res.WasSuccessful() {
log.Warningf("Unable to delete DNS pods: %s", res.OutputPrettyPrint())
}
Expand All @@ -200,7 +196,7 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers
}

testfunc := func() {
By("Deleting Cilium, CoreDNS, and etcd-operator...")
By("Deleting Cilium and CoreDNS...")
// Making sure that we deleted the cilium ds. No assert
// message because maybe is not present
if res := kubectl.DeleteResource("ds", fmt.Sprintf("-n %s cilium", helpers.CiliumNamespace)); !res.WasSuccessful() {
Expand All @@ -213,11 +209,6 @@ func InstallAndValidateCiliumUpgrades(kubectl *helpers.Kubectl, oldHelmChartVers
log.Warningf("Unable to delete DNS pods: %s", res.OutputPrettyPrint())
}

// Delete all etcd pods otherwise they will be kept running but
// the bpf endpoints will be cleaned up when we restart cilium
// with a clean state a couple lines bellow
kubectl.DeleteETCDOperator()

By("Waiting for pods to be terminated..")
ExpectAllPodsTerminated(kubectl)

Expand Down
8 changes: 0 additions & 8 deletions test/k8sT/assertionHelpers.go
View file
Open in desktop
Expand Up @@ -170,11 +170,3 @@ func SkipItIfNoKubeProxy() {
Skip("kube-proxy is disabled (NodePort BPF is enabled). Skipping test.")
}
}

func deleteETCDOperator(kubectl *helpers.Kubectl) {
// Do not assert on success in AfterEach intentionally to avoid
// incomplete teardown.
_ = kubectl.DeleteResource("deploy", fmt.Sprintf("-n %s -l io.cilium/app=etcd-operator", helpers.CiliumNamespace))
_ = kubectl.DeleteResource("pod", fmt.Sprintf("-n %s -l io.cilium/app=etcd-operator", helpers.CiliumNamespace))
_ = kubectl.WaitCleanAllTerminatingPods(helpers.HelperTimeout)
}
6 changes: 6 additions & 0 deletions test/k8sT/manifests.go
View file
Open in desktop
Expand Up @@ -50,4 +50,10 @@ var (
IPSecSecret = helpers.Manifest{
Filename: "ipsec_secret.yaml",
}

StatelessEtcd = helpers.Manifest{
Filename: "etcd-deployment.yaml",
NumPods: 1,
LabelSelector: "name=stateless-etcd",
}
)