Skip to content

Commit

Permalink
Capabilities: drop build/apps APIServices and disable apiservers when…
Browse files Browse the repository at this point in the history 
… capabilities are not enabled

Unless a capability is unknown, install APIService object and
run apiserver for corresponding API only when enabled.
  • Loading branch information
@ingvagabund
ingvagabund committed Aug 3, 2023
1 parent c02a120 commit dc8a55f
Show file tree
Hide file tree
Showing 3 changed files with 259 additions and 27 deletions.
64 changes: 45 additions & 19 deletions pkg/operator/starter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
operatorv1 "github.com/openshift/api/operator/v1"
configv1client "github.com/openshift/client-go/config/clientset/versioned"
configinformers "github.com/openshift/client-go/config/informers/externalversions"
configinformersconfigv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
operatorv1client "github.com/openshift/client-go/operator/clientset/versioned"
operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions"
operatorcontrolplaneclient "github.com/openshift/client-go/operatorcontrolplane/clientset/versioned"
Expand Down Expand Up @@ -38,6 +39,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
Expand All @@ -52,6 +54,19 @@ const (
oauthAPIServerTargetNamespace = "openshift-oauth-apiserver"
)

var apiServiceGroupVersions = []schema.GroupVersion{
// these are all the apigroups we manage
{Group: "apps.openshift.io", Version: "v1"},
{Group: "authorization.openshift.io", Version: "v1"},
{Group: "build.openshift.io", Version: "v1"},
{Group: "image.openshift.io", Version: "v1"},
{Group: "project.openshift.io", Version: "v1"},
{Group: "quota.openshift.io", Version: "v1"},
{Group: "route.openshift.io", Version: "v1"},
{Group: "security.openshift.io", Version: "v1"},
{Group: "template.openshift.io", Version: "v1"},
}

func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
kubeClient, err := kubernetes.NewForConfig(controllerConfig.ProtoKubeConfig)
if err != nil {
Expand Down Expand Up @@ -156,6 +171,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
operatorClient,
operatorConfigClient.OperatorV1(),
configClient.ConfigV1(),
configInformers.Config().V1().ClusterVersions(),
workloadcontroller.CountNodesFuncWrapper(kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Lister()),
workloadcontroller.EnsureAtMostOnePodPerNode,
"openshift-apiserver",
Expand All @@ -180,8 +196,8 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
controllerConfig.EventRecorder,
).WithAPIServiceController(
"openshift-apiserver",
func() (enabled []*apiregistrationv1.APIService, disabled []*apiregistrationv1.APIService, err error) {
return apiServices(), nil, nil
func() ([]*apiregistrationv1.APIService, []*apiregistrationv1.APIService, error) {
return apiServices(configInformers.Config().V1().ClusterVersions())
},
apiregistrationInformers,
apiregistrationv1Client.ApiregistrationV1(),
Expand Down Expand Up @@ -224,6 +240,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
kubeInformersForNamespaces,
operatorConfigInformers.Operator().V1().OpenShiftAPIServers().Informer(),
configInformers.Config().V1().Images().Informer(),
configInformers.Config().V1().ClusterVersions().Informer(),
).WithStaticResourcesController(
"APIServerStaticResources",
v311_00_assets.Asset,
Expand Down Expand Up @@ -383,21 +400,26 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
return nil
}

func apiServices() []*apiregistrationv1.APIService {
var apiServiceGroupVersions = []schema.GroupVersion{
// these are all the apigroups we manage
{Group: "apps.openshift.io", Version: "v1"},
{Group: "authorization.openshift.io", Version: "v1"},
{Group: "build.openshift.io", Version: "v1"},
{Group: "image.openshift.io", Version: "v1"},
{Group: "project.openshift.io", Version: "v1"},
{Group: "quota.openshift.io", Version: "v1"},
{Group: "route.openshift.io", Version: "v1"},
{Group: "security.openshift.io", Version: "v1"},
{Group: "template.openshift.io", Version: "v1"},
func apiServices(clusterVersionInformer configinformersconfigv1.ClusterVersionInformer) ([]*apiregistrationv1.APIService, []*apiregistrationv1.APIService, error) {
clusterVersion, err := clusterVersionInformer.Lister().Get("version")
if err != nil {
return nil, nil, err
}

knownCaps := sets.New[configv1.ClusterVersionCapability](clusterVersion.Status.Capabilities.KnownCapabilities...)
capsEnabled := sets.New[configv1.ClusterVersionCapability](clusterVersion.Status.Capabilities.EnabledCapabilities...)

groupDisabled := make(map[string]bool)
if knownCaps.Has(configv1.ClusterVersionCapabilityBuild) && !capsEnabled.Has(configv1.ClusterVersionCapabilityBuild) {
groupDisabled["build.openshift.io"] = true
}
if knownCaps.Has(configv1.ClusterVersionCapabilityDeploymentConfig) && !capsEnabled.Has(configv1.ClusterVersionCapabilityDeploymentConfig) {
groupDisabled["apps.openshift.io"] = true
}

ret := []*apiregistrationv1.APIService{}
disabled := []*apiregistrationv1.APIService{}
enabled := []*apiregistrationv1.APIService{}

for _, apiServiceGroupVersion := range apiServiceGroupVersions {
obj := &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{
Expand All @@ -418,16 +440,20 @@ func apiServices() []*apiregistrationv1.APIService {
VersionPriority: 15,
},
}
ret = append(ret, obj)
if groupDisabled[apiServiceGroupVersion.Group] {
disabled = append(disabled, obj)
} else {
enabled = append(enabled, obj)
}
}

return ret
return enabled, disabled, nil
}

func apiServicesReferences() []configv1.ObjectReference {
ret := []configv1.ObjectReference{}
for _, apiService := range apiServices() {
ret = append(ret, configv1.ObjectReference{Group: "apiregistration.k8s.io", Resource: "apiservices", Name: apiService.Spec.Version + "." + apiService.Spec.Group})
for _, apiService := range apiServiceGroupVersions {
ret = append(ret, configv1.ObjectReference{Group: "apiregistration.k8s.io", Resource: "apiservices", Name: apiService.Version + "." + apiService.Group})
}
return ret
}
48 changes: 42 additions & 6 deletions pkg/operator/workload/workload_openshiftapiserver_v311_00_sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,18 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/klog/v2"

openshiftapi "github.com/openshift/api"
configv1 "github.com/openshift/api/config/v1"
openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
operatorv1 "github.com/openshift/api/operator/v1"
openshiftconfigclientv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
configinformersconfigv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/operatorclient"
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/v311_00_assets"
Expand All @@ -55,10 +58,11 @@ type ensureAtMostOnePodPerNodeFunc func(spec *appsv1.DeploymentSpec, component s

// OpenShiftAPIServerWorkload is a struct that holds necessary data to install OpenShiftAPIServer
type OpenShiftAPIServerWorkload struct {
operatorClient v1helpers.OperatorClient
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface
kubeClient kubernetes.Interface
operatorClient v1helpers.OperatorClient
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface
clusterVersionInformer configinformersconfigv1.ClusterVersionInformer
kubeClient kubernetes.Interface

// countNodes a function to return count of nodes on which the workload will be installed
countNodes nodeCountFunc
Expand All @@ -79,6 +83,7 @@ func NewOpenShiftAPIServerWorkload(
operatorClient v1helpers.OperatorClient,
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter,
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface,
clusterVersionInformer configinformersconfigv1.ClusterVersionInformer,
countNodes nodeCountFunc,
ensureAtMostOnePodPerNode ensureAtMostOnePodPerNodeFunc,
targetNamespace string,
Expand All @@ -91,6 +96,7 @@ func NewOpenShiftAPIServerWorkload(
operatorClient: operatorClient,
operatorConfigClient: operatorConfigClient,
openshiftConfigClient: openshiftConfigClient,
clusterVersionInformer: clusterVersionInformer,
countNodes: countNodes,
ensureAtMostOnePodPerNode: ensureAtMostOnePodPerNode,
targetNamespace: targetNamespace,
Expand Down Expand Up @@ -147,7 +153,7 @@ func (c *OpenShiftAPIServerWorkload) Sync(ctx context.Context, syncContext facto
}
operatorConfig := originalOperatorConfig.DeepCopy()

_, _, err = manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx, c.kubeClient.CoreV1(), syncContext.Recorder(), operatorConfig)
_, _, err = manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx, c.kubeClient.CoreV1(), c.clusterVersionInformer, syncContext.Recorder(), operatorConfig)
if err != nil {
errors = append(errors, fmt.Errorf("%q: %v", "configmap", err))
}
Expand Down Expand Up @@ -273,15 +279,45 @@ func manageOpenShiftAPIServerImageImportCA_v311_00_to_latest(ctx context.Context
return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap)
}

func manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx context.Context, client coreclientv1.ConfigMapsGetter, recorder events.Recorder, operatorConfig *operatorv1.OpenShiftAPIServer) (*corev1.ConfigMap, bool, error) {
func manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx context.Context, client coreclientv1.ConfigMapsGetter, clusterVersionInformer configinformersconfigv1.ClusterVersionInformer, recorder events.Recorder, operatorConfig *operatorv1.OpenShiftAPIServer) (*corev1.ConfigMap, bool, error) {
configMap := resourceread.ReadConfigMapV1OrDie(v311_00_assets.MustAsset("v3.11.0/openshift-apiserver/cm.yaml"))
defaultConfig := v311_00_assets.MustAsset("v3.11.0/config/defaultconfig.yaml")

clusterVersion, err := clusterVersionInformer.Lister().Get("version")
if err != nil {
return nil, false, err
}

knownCaps := sets.New[configv1.ClusterVersionCapability](clusterVersion.Status.Capabilities.KnownCapabilities...)
capsEnabled := sets.New[configv1.ClusterVersionCapability](clusterVersion.Status.Capabilities.EnabledCapabilities...)

apiServersConfig := openshiftcontrolplanev1.OpenShiftAPIServerConfig{
APIServers: openshiftcontrolplanev1.APIServers{
PerGroupOptions: []openshiftcontrolplanev1.PerGroupOptions{},
},
}
if knownCaps.Has(configv1.ClusterVersionCapabilityBuild) && !capsEnabled.Has(configv1.ClusterVersionCapabilityBuild) {
klog.Infof("Capability %q not enabled, disabling 'openshift.io/build' controller", configv1.ClusterVersionCapabilityBuild)
apiServersConfig.APIServers.PerGroupOptions = append(apiServersConfig.APIServers.PerGroupOptions, openshiftcontrolplanev1.PerGroupOptions{Name: openshiftcontrolplanev1.OpenShiftBuildAPIserver, DisabledVersions: []string{"v1"}})
}

if knownCaps.Has(configv1.ClusterVersionCapabilityDeploymentConfig) && !capsEnabled.Has(configv1.ClusterVersionCapabilityDeploymentConfig) {
klog.Infof("Capability %q not enabled, disabling 'openshift.io/apps' controller", configv1.ClusterVersionCapabilityDeploymentConfig)
apiServersConfig.APIServers.PerGroupOptions = append(apiServersConfig.APIServers.PerGroupOptions, openshiftcontrolplanev1.PerGroupOptions{Name: openshiftcontrolplanev1.OpenShiftAppsAPIserver, DisabledVersions: []string{"v1"}})
}

bytes, err := json.Marshal(apiServersConfig)
if err != nil {
return nil, false, fmt.Errorf("unable to marshal OpenShiftAPIServerConfig struct: %v", err)
}

requiredConfigMap, _, err := resourcemerge.MergePrunedConfigMap(
&openshiftcontrolplanev1.OpenShiftAPIServerConfig{},
configMap,
"config.yaml",
nil,
defaultConfig,
bytes,
operatorConfig.Spec.ObservedConfig.Raw,
operatorConfig.Spec.UnsupportedConfigOverrides.Raw,
)
Expand Down

0 comments on commit dc8a55f

Please sign in to comment.