feat: add cilium e2e tests

Add cilium e2e tests. The existing cilium check was very old, update to latest cilium version and also add a test for KPR strict mode. Signed-off-by: Noel Georgi <git@frezbo.dev>
siderolabs · Mar 3, 2023 · a782812 · a782812
1 parent 061640c
commit a782812
Show file tree

Hide file tree

Showing 7 changed files with 232 additions and 149 deletions.
diff --git a/.drone.jsonnet b/.drone.jsonnet
@@ -416,17 +416,28 @@ local integration_provision_tests_track_2 = Step('provision-tests-track-2', priv
 local integration_extensions = Step('e2e-extensions', target='e2e-qemu', privileged=true, depends_on=[extensions_patch_manifest], environment={
   SHORT_INTEGRATION_TEST: 'yes',
   QEMU_MEMORY_WORKERS: '3072',
-  WITH_CONFIG_PATCH_FILE_WORKER: '_out/extensions-patch.json',
+  WITH_CONFIG_PATCH_WORKER: '@_out/extensions-patch.json',
   WITH_TEST: 'run_extensions_test',
   IMAGE_REGISTRY: local_registry,
 });
-local integration_cilium = Step('e2e-cilium-1.9.10', target='e2e-qemu', privileged=true, depends_on=[integration_extensions], environment={
+local integration_cilium = Step('e2e-cilium', target='e2e-qemu', privileged=true, depends_on=[load_artifacts], environment={
   SHORT_INTEGRATION_TEST: 'yes',
-  CUSTOM_CNI_URL: 'https://raw.githubusercontent.com/cilium/cilium/v1.9.10/install/kubernetes/quick-install.yaml',
-  WITH_CONFIG_PATCH: '[{"op": "replace", "path": "/cluster/network/podSubnets", "value": ["10.0.0.0/8"]}]',  // use Pod CIDRs as hardcoded in Cilium's quick-install
+  WITH_SKIP_BOOT_PHASE_FINISHED_CHECK: 'yes',
+  CUSTOM_CNI_NAME: 'cilium',
+  QEMU_WORKERS: '2',
+  WITH_CONFIG_PATCH: '[{"op": "add", "path": "/cluster/network", "value": {"cni": {"name": "none"}}}]',
   IMAGE_REGISTRY: local_registry,
 });
-local integration_canal_reset = Step('e2e-canal-reset', target='e2e-qemu', privileged=true, depends_on=[integration_cilium], environment={
+local integration_cilium_strict = Step('e2e-cilium-strict', target='e2e-qemu', privileged=true, depends_on=[integration_cilium], environment={
+  SHORT_INTEGRATION_TEST: 'yes',
+  WITH_SKIP_BOOT_PHASE_FINISHED_CHECK: 'yes',
+  CUSTOM_CNI_NAME: 'cilium',
+  QEMU_WORKERS: '2',
+  CILIUM_INSTALL_TYPE: 'strict',
+  WITH_CONFIG_PATCH: '[{"op": "add", "path": "/cluster/network", "value": {"cni": {"name": "none"}}}, {"op": "add", "path": "/cluster/proxy", "value": {"disabled": true}}]',
+  IMAGE_REGISTRY: local_registry,
+});
+local integration_canal_reset = Step('e2e-canal-reset', target='e2e-qemu', privileged=true, depends_on=[load_artifacts], environment={
   INTEGRATION_TEST_RUN: 'TestIntegration/api.ResetSuite/TestResetWithSpec',
   CUSTOM_CNI_URL: 'https://raw.githubusercontent.com/projectcalico/calico/v3.25.0/manifests/canal.yaml',
   REGISTRY: local_registry,
@@ -522,7 +533,6 @@ local integration_pipelines = [
   Pipeline('integration-provision-1', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_1]) + integration_trigger(['integration-provision', 'integration-provision-1']),
   Pipeline('integration-provision-2', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_2]) + integration_trigger(['integration-provision', 'integration-provision-2']),
   Pipeline('integration-misc', default_pipeline_steps + [
-    integration_cilium,
     integration_canal_reset,
     integration_bios_cgroupsv1,
     integration_disk_image,
@@ -532,6 +542,7 @@ local integration_pipelines = [
     integration_default_hostname,
   ]) + integration_trigger(['integration-misc']),
   Pipeline('integration-extensions', default_pipeline_steps + [extensions_build, extensions_artifacts, extensions_patch_manifest, integration_extensions]) + integration_trigger(['integration-extensions']),
+  Pipeline('integration-cilium', default_pipeline_steps + [integration_cilium, integration_cilium_strict]) + integration_trigger(['integration-cilium']),
   Pipeline('integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip]) + integration_trigger(['integration-qemu-encrypted-vip']),
   Pipeline('integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race]) + integration_trigger(['integration-qemu-race']),
   Pipeline('integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi]) + integration_trigger(['integration-qemu-csi']),
@@ -543,11 +554,6 @@ local integration_pipelines = [
   Pipeline('cron-integration-provision-1', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_1], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-provision-2', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_2], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-misc', default_pipeline_steps + [
-    extensions_build,
-    extensions_artifacts,
-    extensions_patch_manifest,
-    integration_extensions,
-    integration_cilium,
     integration_canal_reset,
     integration_bios_cgroupsv1,
     integration_disk_image,
@@ -556,6 +562,8 @@ local integration_pipelines = [
     integration_kubespan,
     integration_default_hostname,
   ], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
+  Pipeline('cron-integration-extensions', default_pipeline_steps + [extensions_build, extensions_artifacts, extensions_patch_manifest, integration_extensions], [default_cron_pipeline]) + cron_trigger(['nightly']),
+  Pipeline('cron-integration-cilium', default_pipeline_steps + [integration_cilium, integration_cilium_strict], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']),

diff --git a/Makefile b/Makefile
@@ -55,10 +55,13 @@ KUBESTR_VERSION ?= v0.4.37
 HELM_VERSION ?= v3.11.1
 # renovate: datasource=github-releases depName=kubernetes-sigs/cluster-api
 CLUSTERCTL_VERSION ?= 1.3.3
+# renovate: datasource=github-releases depName=cilium/cilium-cli
+CILIUM_CLI_VERSION ?= v0.13.0
 KUBECTL_URL ?= https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(OPERATING_SYSTEM)/amd64/kubectl
 KUBESTR_URL ?= https://github.com/kastenhq/kubestr/releases/download/$(KUBESTR_VERSION)/kubestr_$(subst v,,$(KUBESTR_VERSION))_Linux_amd64.tar.gz
 HELM_URL ?= https://get.helm.sh/helm-$(HELM_VERSION)-linux-amd64.tar.gz
 CLUSTERCTL_URL ?= https://github.com/kubernetes-sigs/cluster-api/releases/download/v$(CLUSTERCTL_VERSION)/clusterctl-$(OPERATING_SYSTEM)-amd64
+CILIUM_CLI_URL ?= https://github.com/cilium/cilium-cli/releases/download/$(CILIUM_CLI_VERSION)/cilium-$(OPERATING_SYSTEM)-amd64.tar.gz
 TESTPKGS ?= github.com/siderolabs/talos/...
 RELEASES ?= v1.1.2 v1.2.6
 SHORT_INTEGRATION_TEST ?=
@@ -372,7 +375,12 @@ $(ARTIFACTS)/clusterctl:
 	@curl -L -o $(ARTIFACTS)/clusterctl "$(CLUSTERCTL_URL)"
 	@chmod +x $(ARTIFACTS)/clusterctl
 
-e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr $(ARTIFACTS)/helm ## Runs the E2E test for the specified platform (e.g. e2e-docker).
+$(ARTIFACTS)/cilium:
+	@mkdir -p $(ARTIFACTS)
+	@curl -L "$(CILIUM_CLI_URL)" | tar xzf - -C $(ARTIFACTS) cilium
+	@chmod +x $(ARTIFACTS)/cilium
+
+e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr $(ARTIFACTS)/helm $(ARTIFACTS)/cilium ## Runs the E2E test for the specified platform (e.g. e2e-docker).
 	@$(MAKE) hack-test-$@ \
 		PLATFORM=$* \
 		TAG=$(TAG) \
@@ -388,7 +396,8 @@ e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubect
 		KUBECTL=$(PWD)/$(ARTIFACTS)/kubectl \
 		KUBESTR=$(PWD)/$(ARTIFACTS)/kubestr \
 		HELM=$(PWD)/$(ARTIFACTS)/helm \
-		CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl
+		CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl \
+		CILIUM_CLI=$(PWD)/$(ARTIFACTS)/cilium
 
 provision-tests-prepare: release-artifacts $(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64
 

diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go
@@ -85,69 +85,70 @@ const (
 )
 
 var (
-	talosconfig               string
-	nodeImage                 string
-	nodeInstallImage          string
-	registryMirrors           []string
-	registryInsecure          []string
-	kubernetesVersion         string
-	nodeVmlinuzPath           string
-	nodeInitramfsPath         string
-	nodeISOPath               string
-	nodeDiskImagePath         string
-	applyConfigEnabled        bool
-	bootloaderEnabled         bool
-	uefiEnabled               bool
-	extraUEFISearchPaths      []string
-	configDebug               bool
-	networkCIDR               string
-	networkMTU                int
-	networkIPv4               bool
-	networkIPv6               bool
-	wireguardCIDR             string
-	nameservers               []string
-	dnsDomain                 string
-	workers                   int
-	controlplanes             int
-	controlPlaneCpus          string
-	workersCpus               string
-	controlPlaneMemory        int
-	workersMemory             int
-	clusterDiskSize           int
-	clusterDisks              []string
-	extraDisks                int
-	extraDiskSize             int
-	targetArch                string
-	clusterWait               bool
-	clusterWaitTimeout        time.Duration
-	forceInitNodeAsEndpoint   bool
-	forceEndpoint             string
-	inputDir                  string
-	cniBinPath                []string
-	cniConfDir                string
-	cniCacheDir               string
-	cniBundleURL              string
-	ports                     string
-	dockerHostIP              string
-	withInitNode              bool
-	customCNIUrl              string
-	crashdumpOnFailure        bool
-	skipKubeconfig            bool
-	skipInjectingConfig       bool
-	talosVersion              string
-	encryptStatePartition     bool
-	encryptEphemeralPartition bool
-	useVIP                    bool
-	enableKubeSpan            bool
-	enableClusterDiscovery    bool
-	configPatch               []string
-	configPatchControlPlane   []string
-	configPatchWorker         []string
-	badRTC                    bool
-	extraBootKernelArgs       string
-	dockerDisableIPv6         bool
-	controlPlanePort          int
-	dhcpSkipHostname          bool
+	talosconfig                string
+	nodeImage                  string
+	nodeInstallImage           string
+	registryMirrors            []string
+	registryInsecure           []string
+	kubernetesVersion          string
+	nodeVmlinuzPath            string
+	nodeInitramfsPath          string
+	nodeISOPath                string
+	nodeDiskImagePath          string
+	applyConfigEnabled         bool
+	bootloaderEnabled          bool
+	uefiEnabled                bool
+	extraUEFISearchPaths       []string
+	configDebug                bool
+	networkCIDR                string
+	networkMTU                 int
+	networkIPv4                bool
+	networkIPv6                bool
+	wireguardCIDR              string
+	nameservers                []string
+	dnsDomain                  string
+	workers                    int
+	controlplanes              int
+	controlPlaneCpus           string
+	workersCpus                string
+	controlPlaneMemory         int
+	workersMemory              int
+	clusterDiskSize            int
+	clusterDisks               []string
+	extraDisks                 int
+	extraDiskSize              int
+	targetArch                 string
+	clusterWait                bool
+	clusterWaitTimeout         time.Duration
+	forceInitNodeAsEndpoint    bool
+	forceEndpoint              string
+	inputDir                   string
+	cniBinPath                 []string
+	cniConfDir                 string
+	cniCacheDir                string
+	cniBundleURL               string
+	ports                      string
+	dockerHostIP               string
+	withInitNode               bool
+	customCNIUrl               string
+	crashdumpOnFailure         bool
+	skipKubeconfig             bool
+	skipInjectingConfig        bool
+	talosVersion               string
+	encryptStatePartition      bool
+	encryptEphemeralPartition  bool
+	useVIP                     bool
+	enableKubeSpan             bool
+	enableClusterDiscovery     bool
+	configPatch                []string
+	configPatchControlPlane    []string
+	configPatchWorker          []string
+	badRTC                     bool
+	extraBootKernelArgs        string
+	dockerDisableIPv6          bool
+	controlPlanePort           int
+	dhcpSkipHostname           bool
+	skipBootPhaseFinishedCheck bool
 )
 
 // createCmd represents the cluster up command.
@@ -693,7 +694,15 @@ func postCreate(ctx context.Context, clusterAccess *access.Adapter) error {
 	checkCtx, checkCtxCancel := context.WithTimeout(ctx, clusterWaitTimeout)
 	defer checkCtxCancel()
 
-	if err := check.Wait(checkCtx, clusterAccess, append(check.DefaultClusterChecks(), check.ExtraClusterChecks()...), check.StderrReporter()); err != nil {
+	checks := check.DefaultClusterChecks()
+
+	if skipBootPhaseFinishedCheck {
+		checks = check.PreBootSequenceChecks()
+	}
+
+	checks = append(checks, check.ExtraClusterChecks()...)
+
+	if err := check.Wait(checkCtx, clusterAccess, checks, check.StderrReporter()); err != nil {
 		return err
 	}
 
@@ -929,6 +938,7 @@ func init() {
 	createCmd.Flags().BoolVar(&dockerDisableIPv6, "docker-disable-ipv6", false, "skip enabling IPv6 in containers (Docker only)")
 	createCmd.Flags().IntVar(&controlPlanePort, controlPlanePortFlag, constants.DefaultControlPlanePort, "control plane port (load balancer and local API port)")
 	createCmd.Flags().BoolVar(&dhcpSkipHostname, "disable-dhcp-hostname", false, "skip announcing hostname via DHCP (QEMU only)")
+	createCmd.Flags().BoolVar(&skipBootPhaseFinishedCheck, "skip-boot-phase-finished-check", false, "skip waiting for node to finish boot phase")
 
 	Cmd.AddCommand(createCmd)
 }

diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh
@@ -102,31 +102,30 @@ esac
 case "${WITH_CONFIG_PATCH:-false}" in
   # using arrays here to preserve spaces properly in WITH_CONFIG_PATCH
   false)
-     CONFIG_PATCH_FLAG=()
-    ;;
+      CONFIG_PATCH_FLAG=()
+      ;;
   *)
-    CONFIG_PATCH_FLAG=(--config-patch "${WITH_CONFIG_PATCH}")
-    ;;
+      CONFIG_PATCH_FLAG=(--config-patch "${WITH_CONFIG_PATCH}")
+      ;;
 esac
 
-case "${WITH_CONFIG_PATCH_FILE:-false}" in
-  # using arrays here to preserve spaces properly in WITH_CONFIG_PATCH_FILE
+case "${WITH_CONFIG_PATCH_WORKER:-false}" in
+  # using arrays here to preserve spaces properly in WITH_CONFIG_PATCH_WORKER
   false)
-     CONFIG_PATCH_FLAG=()
-    ;;
+      CONFIG_PATCH_FLAG=()
+      ;;
   *)
-    CONFIG_PATCH_FLAG=(--config-patch "@${WITH_CONFIG_PATCH_FILE}")
-    ;;
+      CONFIG_PATCH_FLAG=(--config-patch-worker "${WITH_CONFIG_PATCH_FILE}")
+      ;;
 esac
 
-case "${WITH_CONFIG_PATCH_FILE_WORKER:-false}" in
-  # using arrays here to preserve spaces properly in WITH_CONFIG_PATCH_FILE
-  false)
-     CONFIG_PATCH_FLAG=()
-    ;;
+case "${WITH_SKIP_BOOT_PHASE_FINISHED_CHECK:-no}" in
+  yes|true|y)
+      SKIP_BOOT_PHASE_FINISHED_CHECK_FLAG="--skip-boot-phase-finished-check=true"
+      ;;
   *)
-    CONFIG_PATCH_FLAG=(--config-patch-worker "@${WITH_CONFIG_PATCH_FILE_WORKER}")
-    ;;
+      SKIP_BOOT_PHASE_FINISHED_CHECK_FLAG="--skip-boot-phase-finished-check=false"
+      ;;
 esac
 
 function create_cluster {
@@ -158,7 +157,8 @@ function create_cluster {
     ${REGISTRY_MIRROR_FLAGS} \
     ${QEMU_FLAGS} \
     ${CUSTOM_CNI_FLAG} \
-    "${CONFIG_PATCH_FLAG[@]}"
+    "${CONFIG_PATCH_FLAG[@]}" \
+    "${SKIP_BOOT_PHASE_FINISHED_CHECK_FLAG}"
 
   "${TALOSCTL}" config node 172.20.1.2
 }
@@ -169,6 +169,14 @@ function destroy_cluster() {
 
 create_cluster
 
+case "${CUSTOM_CNI_NAME:-none}" in
+  cilium)
+    install_and_run_cilium_cni_tests
+    ;;
+  *)
+    ;;
+esac
+
 case "${TEST_MODE:-default}" in
   fast-conformance)
     run_kubernetes_conformance_test fast