Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove option to disable OSM #13381

Merged
merged 3 commits into from
May 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
43 changes: 0 additions & 43 deletions .prow/features.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -272,46 +272,3 @@ presubmits:
cpu: 2
limits:
memory: 6Gi

- name: pre-kubermatic-legacy-machine-controller-user-data-e2e
decorate: true
clone_uri: "ssh://git@github.com/kubermatic/kubermatic.git"
labels:
preset-aws-e2e-kkp: "true"
preset-docker-mirror: "true"
preset-docker-pull: "true"
preset-docker-push: "true"
preset-vault: "true"
preset-repo-ssh: "true"
preset-e2e-ssh: "true"
preset-kubeconfig-ci: "true"
preset-kind-volume-mounts: "true"
preset-goproxy: "true"
spec:
containers:
- image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-4
env:
- name: KUBERMATIC_EDITION
value: ee
- name: RELEASES_TO_TEST
value: "1.29"
- name: DISTRIBUTIONS
value: ubuntu
- name: KUBERMATIC_OSM_ENABLED
value: "false"
- name: SERVICE_ACCOUNT_KEY
valueFrom:
secretKeyRef:
name: e2e-ci
key: serviceAccountSigningKey
command:
- "./hack/ci/run-e2e-tests.sh"
# docker-in-docker needs privileged mode
securityContext:
privileged: true
resources:
requests:
memory: 4Gi
cpu: 3.5
limits:
memory: 6Gi
2 changes: 0 additions & 2 deletions .prow/provider-anexia.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ presubmits:
value: "anexia"
- name: DISTRIBUTIONS
value: flatcar
- name: KUBERMATIC_OSM_ENABLED
value: "false"
- name: DEFAULT_TIMEOUT_MINUTES
value: "20"
- name: SERVICE_ACCOUNT_KEY
Expand Down
1 change: 0 additions & 1 deletion cmd/conformance-tester/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ _build/conformance-tester \
-providers "aws" \
-distributions "${DISTRIBUTIONS:-}" \
-releases "${RELEASES:-}" \
-enable-osm=${OSM:-true} \
-container-runtimes "${RUNTIMES:-}" \
-client "kube" \
-log-format "Console" \
Expand Down
3 changes: 0 additions & 3 deletions cmd/conformance-tester/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ func main() {
"operatingsystems", sets.List(opts.Distributions),
"versions", opts.Versions,
"tests", sets.List(opts.Tests),
"osm", opts.OperatingSystemManagerEnabled,
"dualstack", opts.DualStackEnabled,
"konnectivity", opts.KonnectivityEnabled,
"updates", opts.TestClusterUpdate,
Expand Down Expand Up @@ -121,7 +120,6 @@ func main() {
scenarios, err := scenarios.NewGenerator().
WithCloudProviders(sets.List(opts.Providers)...).
WithOperatingSystems(sets.List(opts.Distributions)...).
WithOSM(opts.OperatingSystemManagerEnabled).
WithDualstack(opts.DualStackEnabled).
WithVersions(opts.Versions...).
Scenarios(rootCtx, opts, log)
Expand Down Expand Up @@ -346,7 +344,6 @@ func keepOnlyFailedScenarios(log *zap.SugaredLogger, allScenarios []scenarios.Sc

func optionsChanged(previous runner.TestConfiguration, current types.Options) bool {
return false ||
previous.OSMEnabled != current.OperatingSystemManagerEnabled ||
previous.KonnectivityEnabled != current.KonnectivityEnabled ||
previous.DualstackEnabled != current.DualStackEnabled ||
previous.TestClusterUpdate != current.TestClusterUpdate ||
Expand Down
1 change: 0 additions & 1 deletion cmd/conformance-tester/pkg/clients/client_kube.go
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,6 @@ func (c *kubeClient) CreateCluster(ctx context.Context, log *zap.SugaredLogger,

cluster.Spec = *scenario.Cluster(c.opts.Secrets)
cluster.Spec.HumanReadableName = humanReadableName
cluster.Spec.EnableOperatingSystemManager = ptr.To(c.opts.OperatingSystemManagerEnabled)
cluster.Spec.ClusterNetwork.KonnectivityEnabled = ptr.To(c.opts.KonnectivityEnabled) //nolint:staticcheck

if c.opts.DualStackEnabled {
Expand Down
2 changes: 0 additions & 2 deletions cmd/conformance-tester/pkg/runner/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ func (r *Results) PrintSummary() {
fmt.Println("Parameters:")
fmt.Printf(" KKP Version............: %s (%s)\n", r.Options.KubermaticConfiguration.Status.KubermaticVersion, r.Options.KubermaticConfiguration.Status.KubermaticEdition)
fmt.Printf(" Name Prefix............: %q\n", r.Options.NamePrefix)
fmt.Printf(" OSM Enabled............: %v\n", r.Options.OperatingSystemManagerEnabled)
fmt.Printf(" Dualstack Enabled......: %v\n", r.Options.DualStackEnabled)
fmt.Printf(" Konnectivity Enabled...: %v\n", r.Options.KonnectivityEnabled)
fmt.Printf(" Cluster Updates Enabled: %v\n", r.Options.TestClusterUpdate)
Expand Down Expand Up @@ -160,7 +159,6 @@ func (r *Results) WriteToFile(filename string) error {

output := ResultsFile{
Configuration: TestConfiguration{
OSMEnabled: r.Options.OperatingSystemManagerEnabled,
DualstackEnabled: r.Options.DualStackEnabled,
KonnectivityEnabled: r.Options.KonnectivityEnabled,
TestClusterUpdate: r.Options.TestClusterUpdate,
Expand Down
6 changes: 0 additions & 6 deletions cmd/conformance-tester/pkg/scenarios/generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ type Generator struct {
cloudProviders sets.Set[string]
operatingSystems sets.Set[string]
versions sets.Set[string]
enableOSM bool
enableDualstack bool
}

Expand Down Expand Up @@ -70,11 +69,6 @@ func (g *Generator) WithVersions(versions ...*semver.Semver) *Generator {
return g
}

func (g *Generator) WithOSM(enable bool) *Generator {
g.enableOSM = enable
return g
}

func (g *Generator) WithDualstack(enable bool) *Generator {
g.enableDualstack = enable
return g
Expand Down
10 changes: 4 additions & 6 deletions cmd/conformance-tester/pkg/types/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,10 @@ type Options struct {

// additional settings identical for all scenarios

OperatingSystemManagerEnabled bool
DualStackEnabled bool
KonnectivityEnabled bool
ScenarioOptions sets.Set[string]
TestClusterUpdate bool
DualStackEnabled bool
KonnectivityEnabled bool
ScenarioOptions sets.Set[string]
TestClusterUpdate bool

// additional settings

Expand Down Expand Up @@ -157,7 +156,6 @@ func (o *Options) AddFlags() {
flag.BoolVar(&o.DeleteClusterAfterTests, "kubermatic-delete-cluster", true, "delete test cluster when tests where successful")
flag.BoolVar(&o.WaitForClusterDeletion, "wait-for-cluster-deletion", true, "wait for the cluster deletion to have finished")
flag.StringVar(&pubKeyPath, "node-ssh-pub-key", pubKeyPath, "path to a public key which gets deployed onto every node")
flag.BoolVar(&o.OperatingSystemManagerEnabled, "enable-osm", true, "When set, enables Operating System Manager in the user cluster")
flag.BoolVar(&o.DualStackEnabled, "enable-dualstack", false, "When set, enables dualstack (IPv4+IPv6 networking) in the user cluster")
flag.BoolVar(&o.KonnectivityEnabled, "enable-konnectivity", true, "When set, enables Konnectivity (proxy service for control plane communication) in the user cluster. When set to false, OpenVPN is used")
flag.BoolVar(&o.TestClusterUpdate, "update-cluster", false, "When set, will first run the selected tests, then update the cluster and nodes to their next minor release and then run the same tests again")
Expand Down
1 change: 0 additions & 1 deletion hack/ci/run-conformance-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@ timeout -s 9 "${maxDuration}m" ./_build/conformance-tester $EXTRA_ARGS \
-distributions="${DISTRIBUTIONS:-}" \
-exclude-distributions="${EXCLUDE_DISTRIBUTIONS:-}" \
-exclude-tests="${EXCLUDE_TESTS:-}" \
-enable-osm=${KUBERMATIC_OSM_ENABLED:-true} \
-scenario-options="${SCENARIO_OPTIONS:-}" \
-pushgateway-endpoint="pushgateway.monitoring.svc.cluster.local.:9091" \
-results-file "$ARTIFACTS/conformance-tester-results.json"
2 changes: 0 additions & 2 deletions hack/ci/setup-kubermatic-in-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ fi

# The Kubermatic version to build.
export KUBERMATIC_VERSION="${KUBERMATIC_VERSION:-$(git rev-parse HEAD)}"
KUBERMATIC_OSM_ENABLED="${KUBERMATIC_OSM_ENABLED:-true}"

REPOSUFFIX=""
if [ "$KUBERMATIC_EDITION" != "ce" ]; then
Expand Down Expand Up @@ -129,7 +128,6 @@ cp $KUBERMATIC_YAML $KUBERMATIC_CONFIG
sed -i "s;__SERVICE_ACCOUNT_KEY__;$SERVICE_ACCOUNT_KEY;g" $KUBERMATIC_CONFIG
sed -i "s;__IMAGE_PULL_SECRET__;$IMAGE_PULL_SECRET_INLINE;g" $KUBERMATIC_CONFIG
sed -i "s;__KUBERMATIC_DOMAIN__;$KUBERMATIC_DOMAIN;g" $KUBERMATIC_CONFIG
sed -i "s;__KUBERMATIC_OSM_ENABLED__;$KUBERMATIC_OSM_ENABLED;g" $KUBERMATIC_CONFIG

HELM_VALUES_FILE="$(mktemp)"
cat << EOF > $HELM_VALUES_FILE
Expand Down
2 changes: 0 additions & 2 deletions hack/local/run-kubermatic-kind.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ cd $(dirname $0)/../..
source hack/lib.sh

KUBERMATIC_DOMAIN="${KUBERMATIC_DOMAIN:-kubermatic.local}"
KUBERMATIC_OSM_ENABLED="${KUBERMATIC_OSM_ENABLED:-false}"
export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kubermatic}"
export KUBERMATIC_EDITION="${KUBERMATIC_EDITION:-ce}"
export BUILD_ID="${BUILD_ID:-abc}"
Expand Down Expand Up @@ -161,7 +160,6 @@ cp hack/ci/testdata/kubermatic.yaml $KUBERMATIC_CONFIG

sed -i "s;__IMAGE_PULL_SECRET__;$IMAGE_PULL_SECRET_INLINE;g" $KUBERMATIC_CONFIG
sed -i "s;__KUBERMATIC_DOMAIN__;$KUBERMATIC_DOMAIN;g" $KUBERMATIC_CONFIG
sed -i "s;__KUBERMATIC_OSM_ENABLED__;$KUBERMATIC_OSM_ENABLED;g" $KUBERMATIC_CONFIG

HELM_VALUES_FILE="$(mktemp)"
cat << EOF > $HELM_VALUES_FILE
Expand Down
8 changes: 2 additions & 6 deletions pkg/apis/kubermatic/v1/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,8 +214,8 @@ type ClusterSpec struct {
// No SSH keys will be synced after node creation if this is disabled.
EnableUserSSHKeyAgent *bool `json:"enableUserSSHKeyAgent,omitempty"`

// Optional: Enables operating-system-manager (OSM), which is responsible for creating and managing worker node configuration.
// This field is enabled(true) by default.
// Deprecated: EnableOperatingSystemManager has been deprecated starting with KKP 2.26 and will be removed in KKP 2.28+. This field is no-op and OSM is always enabled for user clusters.
// OSM is responsible for creating and managing worker node configuration.
EnableOperatingSystemManager *bool `json:"enableOperatingSystemManager,omitempty"`

// KubeLB holds the configuration for the kubeLB component.
Expand Down Expand Up @@ -265,10 +265,6 @@ type ClusterSpec struct {
BackupConfig *BackupConfig `json:"backupConfig,omitempty"`
}

func (c ClusterSpec) IsOperatingSystemManagerEnabled() bool {
return c.EnableOperatingSystemManager == nil || *c.EnableOperatingSystemManager
}

// KubernetesDashboard contains settings for the kubernetes-dashboard component as part of the cluster control plane.
type KubernetesDashboard struct {
// Controls whether kubernetes-dashboard is deployed to the user cluster or not.
Expand Down
10 changes: 4 additions & 6 deletions pkg/controller/seed-controller-manager/kubernetes/health.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,13 +100,11 @@ func (r *Reconciler) clusterHealth(ctx context.Context, cluster *kubermaticv1.Cl
}
extendedHealth.ApplicationController = kubermaticv1helper.GetHealthStatus(applicationControllerHealthStatus, cluster, r.versions)

if cluster.Spec.IsOperatingSystemManagerEnabled() {
status, err := r.operatingSystemManagerHealthCheck(ctx, cluster, ns)
if err != nil {
return nil, fmt.Errorf("failed to get operating-system-manager health: %w", err)
}
extendedHealth.OperatingSystemManager = &status
status, err := r.operatingSystemManagerHealthCheck(ctx, cluster, ns)
if err != nil {
return nil, fmt.Errorf("failed to get operating-system-manager health: %w", err)
}
extendedHealth.OperatingSystemManager = &status

if cluster.Spec.IsKubernetesDashboardEnabled() {
status, err := r.kubernetesDashboardHealthCheck(ctx, cluster, ns)
Expand Down
45 changes: 8 additions & 37 deletions pkg/controller/seed-controller-manager/kubernetes/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,13 +188,6 @@ func (r *Reconciler) ensureResourcesAreDeployed(ctx context.Context, cluster *ku
}
}

// Ensure that OSM is completely removed, when disabled
if !cluster.Spec.IsOperatingSystemManagerEnabled() {
if err := r.ensureOSMResourcesAreRemoved(ctx, data); err != nil {
return nil, err
}
}

// Ensure that kubernetes-dashboard is completely removed, when disabled
if !cluster.Spec.IsKubernetesDashboardEnabled() {
if err := r.ensureKubernetesDashboardResourcesAreRemoved(ctx, data); err != nil {
Expand Down Expand Up @@ -348,6 +341,7 @@ func GetServiceReconcilers(data *resources.TemplateData) []reconciling.NamedServ
apiserver.ServiceReconciler(data.Cluster().Spec.ExposeStrategy, extName),
etcd.ServiceReconciler(data),
userclusterwebhook.ServiceReconciler(),
operatingsystemmanager.ServiceReconciler(),
}

if data.Cluster().Spec.Cloud.Edge == nil {
Expand All @@ -368,10 +362,6 @@ func GetServiceReconcilers(data *resources.TemplateData) []reconciling.NamedServ
creators = append(creators, nodeportproxy.FrontLoadBalancerServiceReconciler(data))
}

if data.Cluster().Spec.IsOperatingSystemManagerEnabled() {
creators = append(creators, operatingsystemmanager.ServiceReconciler())
}

return creators
}

Expand All @@ -388,6 +378,8 @@ func GetDeploymentReconcilers(data *resources.TemplateData, enableAPIserverOIDCA
controllermanager.DeploymentReconciler(data),
usercluster.DeploymentReconciler(data),
userclusterwebhook.DeploymentReconciler(data),
operatingsystemmanager.DeploymentReconciler(data),
operatingsystemmanager.WebhookDeploymentReconciler(data),
}

// BYO and Edge provider doesn't need machine controller.
Expand Down Expand Up @@ -421,11 +413,6 @@ func GetDeploymentReconcilers(data *resources.TemplateData, enableAPIserverOIDCA
deployments = append(deployments, cloudcontroller.DeploymentReconciler(data))
}

if data.Cluster().Spec.IsOperatingSystemManagerEnabled() {
deployments = append(deployments, operatingsystemmanager.DeploymentReconciler(data))
deployments = append(deployments, operatingsystemmanager.WebhookDeploymentReconciler(data))
}

if data.Cluster().Spec.ExposeStrategy == kubermaticv1.ExposeStrategyLoadBalancer {
deployments = append(deployments,
nodeportproxy.DeploymentEnvoyReconciler(data, versions),
Expand Down Expand Up @@ -502,6 +489,10 @@ func (r *Reconciler) GetSecretReconcilers(ctx context.Context, data *resources.T
apiserver.TokenViewerReconciler(),
apiserver.TokenUsersReconciler(data),
resources.ViewerKubeconfigReconciler(data),

// OSM
resources.GetInternalKubeconfigReconciler(namespace, resources.OperatingSystemManagerWebhookKubeconfigSecretName, resources.OperatingSystemManagerWebhookCertUsername, nil, data, r.log),
operatingsystemmanager.TLSServingCertificateReconciler(data),
}

if data.Cluster().Spec.Cloud.Edge == nil {
Expand All @@ -518,13 +509,6 @@ func (r *Reconciler) GetSecretReconcilers(ctx context.Context, data *resources.T
)
}

if data.Cluster().Spec.IsOperatingSystemManagerEnabled() {
creators = append(creators,
resources.GetInternalKubeconfigReconciler(namespace, resources.OperatingSystemManagerWebhookKubeconfigSecretName, resources.OperatingSystemManagerWebhookCertUsername, nil, data, r.log),
operatingsystemmanager.TLSServingCertificateReconciler(data),
)
}

if data.Cluster().Spec.IsKubeLBEnabled() {
creators = append(creators,
resources.GetInternalKubeconfigReconciler(namespace, resources.KubeLBCCMKubeconfigSecretName, resources.KubeLBCCMCertUsername, nil, data, r.log),
Expand Down Expand Up @@ -587,6 +571,7 @@ func (r *Reconciler) ensureServiceAccounts(ctx context.Context, c *kubermaticv1.
etcd.ServiceAccountReconciler,
usercluster.ServiceAccountReconciler,
userclusterwebhook.ServiceAccountReconciler,
operatingsystemmanager.ServiceAccountReconciler,
}

if c.Spec.Cloud.Edge == nil {
Expand All @@ -598,10 +583,6 @@ func (r *Reconciler) ensureServiceAccounts(ctx context.Context, c *kubermaticv1.
namedServiceAccountReconcilerFactories = append(namedServiceAccountReconcilerFactories, csi.ServiceAccountReconcilers(c)...)
}

if c.Spec.IsOperatingSystemManagerEnabled() {
namedServiceAccountReconcilerFactories = append(namedServiceAccountReconcilerFactories, operatingsystemmanager.ServiceAccountReconciler)
}

if c.Spec.ExposeStrategy == kubermaticv1.ExposeStrategyLoadBalancer {
namedServiceAccountReconcilerFactories = append(namedServiceAccountReconcilerFactories, nodeportproxy.ServiceAccountReconciler)
}
Expand Down Expand Up @@ -927,16 +908,6 @@ func (r *Reconciler) ensureCSIDriverResourcesAreRemoved(ctx context.Context, dat
return nil
}

func (r *Reconciler) ensureOSMResourcesAreRemoved(ctx context.Context, data *resources.TemplateData) error {
for _, resource := range operatingsystemmanager.ResourcesForDeletion(data.Cluster().Status.NamespaceName) {
err := r.Client.Delete(ctx, resource)
if err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to ensure OSM resources are removed/not present: %w", err)
}
}
return nil
}

func (r *Reconciler) ensureOpenVPNSetupIsRemoved(ctx context.Context, data *resources.TemplateData) error {
for _, resource := range openvpn.ResourcesForDeletion(data.Cluster().Status.NamespaceName) {
if err := r.Client.Delete(ctx, resource); err != nil && !apierrors.IsNotFound(err) {
Expand Down