From 4a1183b0834fda539f122d7ee6b2c45697cce59e Mon Sep 17 00:00:00 2001 From: Alexey Palazhchenko Date: Tue, 18 May 2021 15:07:04 +0000 Subject: [PATCH] feat: ship Environment "default" Closes #311. Signed-off-by: Alexey Palazhchenko --- Dockerfile | 3 +- .../api/v1alpha1/environment_types.go | 31 +++++++++ .../samples/metal_v1alpha1_environment.yaml | 30 ++++---- .../controllers/environment_controller.go | 68 ++++++++++++++----- .../internal/ipxe/ipxe_server.go | 22 +++--- app/metal-controller-manager/main.go | 40 +++++++---- .../docs/v0.3/Configuration/environments.md | 24 +++---- .../content/docs/v0.3/Guides/bootstrapping.md | 43 +----------- .../content/docs/v0.3/Guides/first-cluster.md | 4 +- .../content/docs/v0.3/Guides/upgrades.md | 7 +- go.mod | 2 +- go.sum | 3 +- sfyra/cmd/sfyra/cmd/options.go | 4 +- sfyra/go.mod | 2 +- sfyra/go.sum | 3 +- sfyra/pkg/tests/environment.go | 67 ++++++++++++++---- sfyra/pkg/tests/server_class.go | 34 ++++++++-- sfyra/pkg/tests/tests.go | 8 ++- 18 files changed, 250 insertions(+), 145 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2994feb49..1f1a42d43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,9 +121,10 @@ LABEL org.opencontainers.image.source https://github.com/talos-systems/sidero ENTRYPOINT [ "/manager" ] FROM base AS build-metal-controller-manager +ARG TALOS_RELEASE ARG TARGETARCH ARG GO_BUILDFLAGS -RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=${TARGETARCH} go build ${GO_BUILDFLAGS} -ldflags "-s -w" -o /manager ./app/metal-controller-manager +RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=${TARGETARCH} go build ${GO_BUILDFLAGS} -ldflags "-s -w -X main.TalosRelease=${TALOS_RELEASE}" -o /manager ./app/metal-controller-manager RUN chmod +x /manager FROM base AS agent-build-amd64 diff --git a/app/metal-controller-manager/api/v1alpha1/environment_types.go b/app/metal-controller-manager/api/v1alpha1/environment_types.go index a5962deb4..a7a627158 100644 --- a/app/metal-controller-manager/api/v1alpha1/environment_types.go +++ b/app/metal-controller-manager/api/v1alpha1/environment_types.go @@ -5,9 +5,16 @@ package v1alpha1 import ( + "fmt" + "sort" + + "github.com/talos-systems/talos/pkg/machinery/kernel" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// EnvironmentDefault is an automatically created Environment. +const EnvironmentDefault = "default" + type Asset struct { URL string `json:"url,omitempty"` SHA512 string `json:"sha512,omitempty"` @@ -65,6 +72,30 @@ type EnvironmentList struct { Items []Environment `json:"items"` } +// EnvironmentDefaultSpec returns EnvironmentDefault's spec. +func EnvironmentDefaultSpec(talosRelease, apiEndpoint string, apiPort uint16) *EnvironmentSpec { + args := make([]string, 0, len(kernel.DefaultArgs)+6) + args = append(args, kernel.DefaultArgs...) + args = append(args, "console=tty0", "console=ttyS1,115200n8", "earlyprintk=ttyS1,115200n8") + args = append(args, "initrd=initramfs.xz", "talos.platform=metal") + args = append(args, fmt.Sprintf("talos.config=http://%s:%d/configdata?uuid=", apiEndpoint, apiPort)) + sort.Strings(args) + + return &EnvironmentSpec{ + Kernel: Kernel{ + Asset: Asset{ + URL: fmt.Sprintf("https://github.com/talos-systems/talos/releases/download/%s/vmlinuz-amd64", talosRelease), + }, + Args: args, + }, + Initrd: Initrd{ + Asset: Asset{ + URL: fmt.Sprintf("https://github.com/talos-systems/talos/releases/download/%s/initramfs-amd64.xz", talosRelease), + }, + }, + } +} + func init() { SchemeBuilder.Register(&Environment{}, &EnvironmentList{}) } diff --git a/app/metal-controller-manager/config/samples/metal_v1alpha1_environment.yaml b/app/metal-controller-manager/config/samples/metal_v1alpha1_environment.yaml index d401b42eb..2a957d451 100644 --- a/app/metal-controller-manager/config/samples/metal_v1alpha1_environment.yaml +++ b/app/metal-controller-manager/config/samples/metal_v1alpha1_environment.yaml @@ -4,23 +4,25 @@ metadata: name: default spec: kernel: - url: 'https://github.com/talos-systems/talos/releases/download/v0.4.0-alpha.5/vmlinuz' - sha512: '' + url: "https://github.com/talos-systems/talos/releases/download/v0.10.2/vmlinuz-amd64" + sha512: "" args: - - initrd=initramfs.xz - - page_poison=1 - - slab_nomerge - - slub_debug=P - - pti=on - - random.trust_cpu=on - - ima_template=ima-ng + - console=tty0 + - console=ttyS1,115200n8 + - consoleblank=0 + - earlyprintk=ttyS1,115200n8 - ima_appraise=fix - ima_hash=sha512 - - console=tty0 - - console=ttyS0 + - ima_template=ima-ng + - init_on_alloc=1 + - initrd=initramfs.xz + - nvme_core.io_timeout=4294967295 - printk.devkmsg=on + - pti=on + - random.trust_cpu=on + - slab_nomerge= + - talos.config=http://192.168.1.10:8081/configdata?uuid= - talos.platform=metal - - talos.config=http://192.168.1.10:8080/assets/controlplane.yaml initrd: - url: 'https://github.com/talos-systems/talos/releases/download/v0.4.0-alpha.5/initramfs.xz' - sha512: '' + url: "https://github.com/talos-systems/talos/releases/download/v0.10.2/initramfs-amd64.xz" + sha512: "" diff --git a/app/metal-controller-manager/controllers/environment_controller.go b/app/metal-controller-manager/controllers/environment_controller.go index 0206f29bf..e08880793 100644 --- a/app/metal-controller-manager/controllers/environment_controller.go +++ b/app/metal-controller-manager/controllers/environment_controller.go @@ -19,6 +19,7 @@ import ( multierror "github.com/hashicorp/go-multierror" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -30,37 +31,38 @@ import ( // EnvironmentReconciler reconciles a Environment object. type EnvironmentReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme + Log logr.Logger + Scheme *runtime.Scheme + TalosRelease string + APIEndpoint string + APIPort uint16 } // +kubebuilder:rbac:groups=metal.sidero.dev,resources=environments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=metal.sidero.dev,resources=environments/status,verbs=get;update;patch func (r *EnvironmentReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - return r.reconcile(req) -} - -func (r *EnvironmentReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { - return ctrl.NewControllerManagedBy(mgr). - WithOptions(options). - For(&metalv1alpha1.Environment{}). - Complete(r) -} - -func (r *EnvironmentReconciler) reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() l := r.Log.WithValues("environment", req.Name) + l.Info("reconciling") + + //nolint:godox + // TODO: We probably should use admission webhooks instead (or in additional) to prevent + // unwanted edits instead of "fixing" the resource after the fact. + if req.Name == metalv1alpha1.EnvironmentDefault { + if err := ReconcileEnvironmentDefault(ctx, r.Client, r.TalosRelease, r.APIEndpoint, r.APIPort); err != nil { + return ctrl.Result{}, err + } + + // do not return; re-reconcile it to update status + } //nolint:wsl var env metalv1alpha1.Environment if err := r.Get(ctx, req.NamespacedName, &env); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - - return ctrl.Result{}, fmt.Errorf("unable to get environment: %w", err) + l.Error(err, "failed fetching resource") + return ctrl.Result{}, client.IgnoreNotFound(err) } envs := filepath.Join("/var/lib/sidero/env", env.GetName()) @@ -180,6 +182,36 @@ func (r *EnvironmentReconciler) reconcile(req ctrl.Request) (ctrl.Result, error) return ctrl.Result{}, nil } +// ReconcileEnvironmentDefault ensures that Environment "default" exist. +func ReconcileEnvironmentDefault(ctx context.Context, c client.Client, talosRelease, apiEndpoint string, apiPort uint16) error { + key := types.NamespacedName{ + Name: metalv1alpha1.EnvironmentDefault, + } + + env := metalv1alpha1.Environment{} + err := c.Get(ctx, key, &env) + + if apierrors.IsNotFound(err) { + env.Name = metalv1alpha1.EnvironmentDefault + env.Spec = *metalv1alpha1.EnvironmentDefaultSpec(talosRelease, apiEndpoint, apiPort) + + err = c.Create(ctx, &env) + } + + return err +} + +func (r *EnvironmentReconciler) SetupWithManager(mgr ctrl.Manager, options controller.Options) error { + if r.TalosRelease == "" { + return errors.New("TalosRelease is not set") + } + + return ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&metalv1alpha1.Environment{}). + Complete(r) +} + func save(ctx context.Context, asset metalv1alpha1.Asset, file string) error { url := asset.URL diff --git a/app/metal-controller-manager/internal/ipxe/ipxe_server.go b/app/metal-controller-manager/internal/ipxe/ipxe_server.go index 948fcfd6a..6b30ef039 100644 --- a/app/metal-controller-manager/internal/ipxe/ipxe_server.go +++ b/app/metal-controller-manager/internal/ipxe/ipxe_server.go @@ -305,20 +305,20 @@ func newEnvironment(server *metalv1alpha1.Server, serverBinding *infrav1.ServerB func newAgentEnvironment(arch string) *metalv1alpha1.Environment { args := []string{ - "initrd=initramfs.xz", - "page_poison=1", - "slab_nomerge", - "slub_debug=P", - "pti=on", - "panic=30", - "random.trust_cpu=on", - "ima_template=ima-ng", + "console=tty0", + "console=ttyS0", "ima_appraise=fix", "ima_hash=sha512", + "ima_template=ima-ng", + "initrd=initramfs.xz", "ip=dhcp", - "console=tty0", - "console=ttyS0", + "page_poison=1", + "panic=30", "printk.devkmsg=on", + "pti=on", + "random.trust_cpu=on", + "slab_nomerge=", + "slub_debug=P", fmt.Sprintf("%s=%s:%d", constants.AgentEndpointArg, apiEndpoint, apiPort), } @@ -347,7 +347,7 @@ func newAgentEnvironment(arch string) *metalv1alpha1.Environment { func newDefaultEnvironment() (env *metalv1alpha1.Environment, err error) { env = &metalv1alpha1.Environment{} - if err := c.Get(context.Background(), types.NamespacedName{Namespace: "", Name: "default"}, env); err != nil { + if err := c.Get(context.Background(), types.NamespacedName{Namespace: "", Name: metalv1alpha1.EnvironmentDefault}, env); err != nil { return nil, err } diff --git a/app/metal-controller-manager/main.go b/app/metal-controller-manager/main.go index a048c8609..ac61669f9 100644 --- a/app/metal-controller-manager/main.go +++ b/app/metal-controller-manager/main.go @@ -49,6 +49,9 @@ const ( ) var ( + // TalosRelease is set as a build argument. + TalosRelease string + scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("setup") ) @@ -93,6 +96,11 @@ func main() { flag.Parse() + // we can't continue without it + if TalosRelease == "" { + panic("TalosRelease is not set during the build") + } + // workaround for clusterctl not accepting empty value as default value if extraAgentKernelArgs == "-" { extraAgentKernelArgs = "" @@ -102,6 +110,15 @@ func main() { apiEndpoint = "" } + if apiEndpoint == "" { + if endpoint, ok := os.LookupEnv("API_ENDPOINT"); ok { + apiEndpoint = endpoint + } else { + setupLog.Error(fmt.Errorf("no api endpoint found"), "") + os.Exit(1) + } + } + ctrl.SetLogger(zap.New(func(o *zap.Options) { o.Development = true })) @@ -148,9 +165,12 @@ func main() { corev1.EventSource{Component: "sidero-controller-manager"}) if err = (&controllers.EnvironmentReconciler{ - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("Environment"), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Environment"), + Scheme: mgr.GetScheme(), + TalosRelease: TalosRelease, + APIEndpoint: apiEndpoint, + APIPort: uint16(apiPort), }).SetupWithManager(mgr, controller.Options{MaxConcurrentReconciles: defaultMaxConcurrentReconciles}); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Environment") os.Exit(1) @@ -191,15 +211,6 @@ func main() { setupLog.Info("starting iPXE server") - if apiEndpoint == "" { - if endpoint, ok := os.LookupEnv("API_ENDPOINT"); ok { - apiEndpoint = endpoint - } else { - setupLog.Error(fmt.Errorf("no api endpoint found"), "unable to start iPXE server", "controller", "Environment") - os.Exit(1) - } - } - if err := ipxe.RegisterIPXE(httpMux, apiEndpoint, apiPort, extraAgentKernelArgs, apiPort, mgr.GetClient()); err != nil { setupLog.Error(err, "unable to start iPXE server", "controller", "Environment") os.Exit(1) @@ -231,6 +242,11 @@ func main() { os.Exit(1) } + if err = controllers.ReconcileEnvironmentDefault(context.TODO(), k8sClient, TalosRelease, apiEndpoint, uint16(apiPort)); err != nil { + setupLog.Error(err, `failed to reconcile Environment "default"`) + os.Exit(1) + } + setupLog.Info("starting manager and HTTP server") var eg errgroup.Group diff --git a/docs/website/content/docs/v0.3/Configuration/environments.md b/docs/website/content/docs/v0.3/Configuration/environments.md index f860e282d..2f053a605 100644 --- a/docs/website/content/docs/v0.3/Configuration/environments.md +++ b/docs/website/content/docs/v0.3/Configuration/environments.md @@ -16,7 +16,7 @@ The hierarchy from most to least respected is: - `.spec.environmentRef` provided at `Server` level - `.spec.environmentRef` provided at `ServerClass` level -- `"default"` `Environment` created by administrator +- `"default"` `Environment` created automatically and modified by an administrator A sample environment definition looks like this: @@ -30,22 +30,22 @@ spec: url: "https://github.com/talos-systems/talos/releases/download/v0.10.2/vmlinuz-amd64" sha512: "" args: - - init_on_alloc=1 - - init_on_free=1 - - slab_nomerge - - pti=on - - consoleblank=0 - - random.trust_cpu=on - - ima_template=ima-ng - - ima_appraise=fix - - ima_hash=sha512 - console=tty0 - console=ttyS1,115200n8 + - consoleblank=0 - earlyprintk=ttyS1,115200n8 - - panic=0 + - ima_appraise=fix + - ima_hash=sha512 + - ima_template=ima-ng + - init_on_alloc=1 + - initrd=initramfs.xz + - nvme_core.io_timeout=4294967295 - printk.devkmsg=on - - talos.platform=metal + - pti=on + - random.trust_cpu=on + - slab_nomerge= - talos.config=http://$PUBLIC_IP:8081/configdata?uuid= + - talos.platform=metal initrd: url: "https://github.com/talos-systems/talos/releases/download/v0.10.2/initramfs-amd64.xz" sha512: "" diff --git a/docs/website/content/docs/v0.3/Guides/bootstrapping.md b/docs/website/content/docs/v0.3/Guides/bootstrapping.md index e477b0d2f..683064c20 100644 --- a/docs/website/content/docs/v0.3/Guides/bootstrapping.md +++ b/docs/website/content/docs/v0.3/Guides/bootstrapping.md @@ -148,47 +148,6 @@ kubectl patch server 00000000-0000-0000-0000-d05099d33360 --type='json' -p='[{"o For more information on server acceptance, see the [server docs](/docs/v0.3/configuration/servers). -## Create the Default Environment - -We must now create an `Environment` in our bootstrap cluster. -An environment is a CRD that tells the PXE component of Sidero what information to return to nodes that request a PXE boot after completing the registration process above. -Things that can be controlled here are kernel flags and the kernel and init images to use. - -To create a default environment that will use the latest published Talos release, issue the following: - -```bash -cat < 0.9 @@ -65,4 +65,3 @@ Update the `spec.controlPlaneConfig.[controlplane,init].talosVersion` fields to - At this point, any new controlplane or worker machines should receive the newer machine config format and join the cluster successfully. You can also proceed to upgrade existing nodes. - diff --git a/go.mod b/go.mod index 069d4445c..d2e49bccb 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/talos-systems/go-debug v0.2.0 github.com/talos-systems/go-kmsg v0.1.0 github.com/talos-systems/go-procfs v0.0.0-20210108152626-8cbc42d3dc24 - github.com/talos-systems/go-retry v0.2.0 + github.com/talos-systems/go-retry v0.3.0 github.com/talos-systems/go-smbios v0.0.0-20210422124317-d3a32bea731a github.com/talos-systems/net v0.2.1-0.20210212213224-05190541b0fa github.com/talos-systems/talos/pkg/machinery v0.0.0-20210416105550-2b83440d6f7f // v0.9.3 diff --git a/go.sum b/go.sum index c7f0aab28..f340318c8 100644 --- a/go.sum +++ b/go.sum @@ -496,8 +496,9 @@ github.com/talos-systems/go-kmsg v0.1.0/go.mod h1:dppwQn+/mrdvsziGMbXjzfc4E+75oZ github.com/talos-systems/go-procfs v0.0.0-20210108152626-8cbc42d3dc24 h1:fN8vYvlB9XBQ5aImb1vLgR0ZaDwvfZfBMptqkpi3aEg= github.com/talos-systems/go-procfs v0.0.0-20210108152626-8cbc42d3dc24/go.mod h1:ATyUGFQIW8OnbnmvqefZWVPgL9g+CAmXHfkgny21xX8= github.com/talos-systems/go-retry v0.1.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= -github.com/talos-systems/go-retry v0.2.0 h1:YpQHmtTZ2k0i/bBYRIasdVmF0XaiISVJUOrmZ6FzgLU= github.com/talos-systems/go-retry v0.2.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= +github.com/talos-systems/go-retry v0.3.0 h1:+fATpp942Rhu8pAefCXjuOApwt5LmhyZNTgLqyANlro= +github.com/talos-systems/go-retry v0.3.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= github.com/talos-systems/go-smbios v0.0.0-20210422124317-d3a32bea731a h1:uUAH6oFZwHdWRlHyBIsM8SEYU4fLM6KGu6bmPZOUKd8= github.com/talos-systems/go-smbios v0.0.0-20210422124317-d3a32bea731a/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8= github.com/talos-systems/goipmi v0.0.0-20210504182258-b54796c8d678 h1:udj668k0XBRUBVATjkgjUfW25OS4/aH9sZnMQkmCOXE= diff --git a/sfyra/cmd/sfyra/cmd/options.go b/sfyra/cmd/sfyra/cmd/options.go index 13bf8c65a..36bad9010 100644 --- a/sfyra/cmd/sfyra/cmd/options.go +++ b/sfyra/cmd/sfyra/cmd/options.go @@ -46,9 +46,7 @@ type Options struct { } // TalosRelease is set as build argument. -var ( - TalosRelease string -) +var TalosRelease string // DefaultOptions returns default settings. func DefaultOptions() Options { diff --git a/sfyra/go.mod b/sfyra/go.mod index a20317111..64ac4b312 100644 --- a/sfyra/go.mod +++ b/sfyra/go.mod @@ -20,7 +20,7 @@ require ( github.com/talos-systems/go-debug v0.2.0 github.com/talos-systems/go-loadbalancer v0.1.1 github.com/talos-systems/go-procfs v0.0.0-20210108152626-8cbc42d3dc24 - github.com/talos-systems/go-retry v0.2.1-0.20210119124456-b9dc1a990133 + github.com/talos-systems/go-retry v0.3.0 github.com/talos-systems/net v0.2.1-0.20210212213224-05190541b0fa github.com/talos-systems/sidero v0.0.0-00010101000000-000000000000 github.com/talos-systems/talos v0.9.3 diff --git a/sfyra/go.sum b/sfyra/go.sum index 85bb8beae..ee01dc542 100644 --- a/sfyra/go.sum +++ b/sfyra/go.sum @@ -970,8 +970,9 @@ github.com/talos-systems/go-procfs v0.0.0-20210108152626-8cbc42d3dc24/go.mod h1: github.com/talos-systems/go-retry v0.1.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= github.com/talos-systems/go-retry v0.1.1-0.20201113203059-8c63d290a688/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= github.com/talos-systems/go-retry v0.2.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= -github.com/talos-systems/go-retry v0.2.1-0.20210119124456-b9dc1a990133 h1:mHnKEViee9x2A6YbsUykwqh7L+tLpm5HTlos2QDlqts= github.com/talos-systems/go-retry v0.2.1-0.20210119124456-b9dc1a990133/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= +github.com/talos-systems/go-retry v0.3.0 h1:+fATpp942Rhu8pAefCXjuOApwt5LmhyZNTgLqyANlro= +github.com/talos-systems/go-retry v0.3.0/go.mod h1:HiXQqyVStZ35uSY/MTLWVvQVmC3lIW2MS5VdDaMtoKM= github.com/talos-systems/go-smbios v0.0.0-20201228201610-fb425d4727e6/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8= github.com/talos-systems/go-smbios v0.0.0-20210422124317-d3a32bea731a/go.mod h1:HxhrzAoTZ7ed5Z5VvtCvnCIrOxyXDS7V2B5hCetAMW8= github.com/talos-systems/grpc-proxy v0.2.0/go.mod h1:sm97Vc/z2cok3pu6ruNeszQej4KDxFrDgfWs4C1mtC4= diff --git a/sfyra/pkg/tests/environment.go b/sfyra/pkg/tests/environment.go index 9ee5e7bda..af382826b 100644 --- a/sfyra/pkg/tests/environment.go +++ b/sfyra/pkg/tests/environment.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/talos-systems/go-procfs/procfs" "github.com/talos-systems/go-retry/retry" @@ -23,10 +24,57 @@ import ( "github.com/talos-systems/sidero/sfyra/pkg/talos" ) -const environmentName = "default" +const environmentName = "sfyra" -// TestEnvironmentDefault verifies default environment creation. +func isEnvironmentReady(env *v1alpha1.Environment) bool { + assetURLs := map[string]struct{}{ + env.Spec.Kernel.URL: {}, + env.Spec.Initrd.URL: {}, + } + + for _, cond := range env.Status.Conditions { + if cond.Status == "True" && cond.Type == "Ready" { + delete(assetURLs, cond.URL) + } + } + + return len(assetURLs) == 0 +} + +// TestEnvironmentDefault verifies environment "default". func TestEnvironmentDefault(ctx context.Context, metalClient client.Client, cluster talos.Cluster, kernelURL, initrdURL string) TestFunc { + return func(t *testing.T) { + var environment v1alpha1.Environment + err := metalClient.Get(ctx, types.NamespacedName{Name: v1alpha1.EnvironmentDefault}, &environment) + require.NoError(t, err) + assert.True(t, isEnvironmentReady(&environment)) + + // delete environment to see it being recreated + err = metalClient.Delete(ctx, &environment) + require.NoError(t, err) + + environment = v1alpha1.Environment{} + err = retry.Constant(30 * time.Second).Retry(func() error { + if err := metalClient.Get(ctx, types.NamespacedName{Name: v1alpha1.EnvironmentDefault}, &environment); err != nil { + if apierrors.IsNotFound(err) { + return retry.ExpectedError(err) + } + return err + } + + if !isEnvironmentReady(&environment) { + return retry.ExpectedErrorf("some assets are not ready") + } + + return nil + }) + require.NoError(t, err) + assert.True(t, isEnvironmentReady(&environment)) + } +} + +// TestEnvironmentCreate verifies environment creation. +func TestEnvironmentCreate(ctx context.Context, metalClient client.Client, cluster talos.Cluster, kernelURL, initrdURL string) TestFunc { return func(t *testing.T) { var environment v1alpha1.Environment @@ -63,19 +111,8 @@ func TestEnvironmentDefault(ctx context.Context, metalClient client.Client, clus return retry.UnexpectedError(err) } - assetURLs := map[string]struct{}{ - kernelURL: {}, - initrdURL: {}, - } - - for _, cond := range environment.Status.Conditions { - if cond.Status == "True" && cond.Type == "Ready" { - delete(assetURLs, cond.URL) - } - } - - if len(assetURLs) > 0 { - return retry.ExpectedError(fmt.Errorf("some assets are not ready: %v", assetURLs)) + if !isEnvironmentReady(&environment) { + return retry.ExpectedErrorf("some assets are not ready") } return nil diff --git a/sfyra/pkg/tests/server_class.go b/sfyra/pkg/tests/server_class.go index 8f86adba4..e74ac5428 100644 --- a/sfyra/pkg/tests/server_class.go +++ b/sfyra/pkg/tests/server_class.go @@ -43,20 +43,44 @@ const ( workloadServerClassName = "workload" ) +// TestServerClassAny verifies server class "any". func TestServerClassAny(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { return func(t *testing.T) { + numNodes := len(vmSet.Nodes()) + var serverClass v1alpha1.ServerClass err := metalClient.Get(ctx, types.NamespacedName{Name: v1alpha1.ServerClassAny}, &serverClass) require.NoError(t, err) assert.Empty(t, serverClass.Spec.Qualifiers) + assert.Len(t, append(serverClass.Status.ServersAvailable, serverClass.Status.ServersInUse...), numNodes) - numNodes := len(vmSet.Nodes()) + // delete server class to see it being recreated + err = metalClient.Delete(ctx, &serverClass) + require.NoError(t, err) + + serverClass = v1alpha1.ServerClass{} + err = retry.Constant(10 * time.Second).Retry(func() error { + if err := metalClient.Get(ctx, types.NamespacedName{Name: v1alpha1.ServerClassAny}, &serverClass); err != nil { + if apierrors.IsNotFound(err) { + return retry.ExpectedError(err) + } + return err + } + + if len(serverClass.Status.ServersAvailable)+len(serverClass.Status.ServersInUse) != numNodes { + return retry.ExpectedErrorf("%d + %d != %d", len(serverClass.Status.ServersAvailable), len(serverClass.Status.ServersInUse), numNodes) + } + + return nil + }) + require.NoError(t, err) + assert.Empty(t, serverClass.Spec.Qualifiers) assert.Len(t, append(serverClass.Status.ServersAvailable, serverClass.Status.ServersInUse...), numNodes) } } -// TestServerClassDefault verifies server class creation. -func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { +// TestServerClassCreate verifies server class creation. +func TestServerClassCreate(ctx context.Context, metalClient client.Client, vmSet *vm.Set) TestFunc { return func(t *testing.T) { classSpec := v1alpha1.ServerClassSpec{ Qualifiers: v1alpha1.Qualifiers{ @@ -80,7 +104,7 @@ func TestServerClassDefault(ctx context.Context, metalClient client.Client, vmSe } if len(serverClass.Status.ServersAvailable)+len(serverClass.Status.ServersInUse) != numNodes { - return retry.ExpectedError(fmt.Errorf("%d + %d != %d", len(serverClass.Status.ServersAvailable), len(serverClass.Status.ServersInUse), numNodes)) + return retry.ExpectedErrorf("%d + %d != %d", len(serverClass.Status.ServersAvailable), len(serverClass.Status.ServersInUse), numNodes) } return nil @@ -233,7 +257,7 @@ func TestServerClassPatch(ctx context.Context, metalClient client.Client, cluste } if response.StatusCode != http.StatusOK { - return retry.ExpectedError(fmt.Errorf("metadata not yet present: %d", response.StatusCode)) + return retry.ExpectedErrorf("metadata not yet present: %d", response.StatusCode) } defer response.Body.Close() diff --git a/sfyra/pkg/tests/tests.go b/sfyra/pkg/tests/tests.go index a680489d4..3b9979189 100644 --- a/sfyra/pkg/tests/tests.go +++ b/sfyra/pkg/tests/tests.go @@ -70,13 +70,17 @@ func Run(ctx context.Context, cluster talos.Cluster, vmSet *vm.Set, capiManager "TestEnvironmentDefault", TestEnvironmentDefault(ctx, metalClient, cluster, options.KernelURL, options.InitrdURL), }, + { + "TestEnvironmentCreate", + TestEnvironmentCreate(ctx, metalClient, cluster, options.KernelURL, options.InitrdURL), + }, { "TestServerClassAny", TestServerClassAny(ctx, metalClient, vmSet), }, { - "TestServerClassDefault", - TestServerClassDefault(ctx, metalClient, vmSet), + "TestServerClassCreate", + TestServerClassCreate(ctx, metalClient, vmSet), }, { "TestServerClassPatch",