Skip to content

Commit

Permalink
Capabilities: drop build/apps APIServices and disable apiservers when…
Browse files Browse the repository at this point in the history 
… capabilities are not enabled

Unless a capability is unknown, install APIService object and
run apiserver for corresponding API only when enabled.
  • Loading branch information
@ingvagabund
ingvagabund committed May 21, 2023
1 parent eaaa4e6 commit d54a619
Show file tree
Hide file tree
Showing 6 changed files with 168 additions and 51 deletions.
99 changes: 65 additions & 34 deletions pkg/operator/starter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
operatorv1 "github.com/openshift/api/operator/v1"
configv1client "github.com/openshift/client-go/config/clientset/versioned"
configinformers "github.com/openshift/client-go/config/informers/externalversions"
configinformersconfigv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
operatorv1client "github.com/openshift/client-go/operator/clientset/versioned"
operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions"
operatorcontrolplaneclient "github.com/openshift/client-go/operatorcontrolplane/clientset/versioned"
Expand All @@ -28,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
Expand All @@ -42,6 +44,7 @@ import (
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/resourcesynccontroller"
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/v311_00_assets"
operatorworkload "github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/workload"
apiservercontrollerapiservice "github.com/openshift/library-go/pkg/operator/apiserver/controller/apiservice"
workloadcontroller "github.com/openshift/library-go/pkg/operator/apiserver/controller/workload"
apiservercontrollerset "github.com/openshift/library-go/pkg/operator/apiserver/controllerset"
libgoetcd "github.com/openshift/library-go/pkg/operator/configobserver/etcd"
Expand All @@ -51,6 +54,19 @@ const (
oauthAPIServerTargetNamespace = "openshift-oauth-apiserver"
)

var apiServiceGroupVersions = []schema.GroupVersion{
// these are all the apigroups we manage
{Group: "apps.openshift.io", Version: "v1"},
{Group: "authorization.openshift.io", Version: "v1"},
{Group: "build.openshift.io", Version: "v1"},
{Group: "image.openshift.io", Version: "v1"},
{Group: "project.openshift.io", Version: "v1"},
{Group: "quota.openshift.io", Version: "v1"},
{Group: "route.openshift.io", Version: "v1"},
{Group: "security.openshift.io", Version: "v1"},
{Group: "template.openshift.io", Version: "v1"},
}

func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
kubeClient, err := kubernetes.NewForConfig(controllerConfig.ProtoKubeConfig)
if err != nil {
Expand Down Expand Up @@ -132,6 +148,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
operatorClient,
operatorConfigClient.OperatorV1(),
configClient.ConfigV1(),
configInformers.Config().V1().ClusterVersions(),
workloadcontroller.CountNodesFuncWrapper(kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes().Lister()),
workloadcontroller.EnsureAtMostOnePodPerNode,
"openshift-apiserver",
Expand All @@ -156,7 +173,9 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
controllerConfig.EventRecorder,
).WithAPIServiceController(
"openshift-apiserver",
func() ([]*apiregistrationv1.APIService, error) { return apiServices(), nil },
func() ([]apiservercontrollerapiservice.ManagedAPIService, error) {
return apiServices(configInformers.Config().V1().ClusterVersions())
},
apiregistrationInformers,
apiregistrationv1Client.ApiregistrationV1(),
kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace),
Expand Down Expand Up @@ -356,51 +375,63 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
return nil
}

func apiServices() []*apiregistrationv1.APIService {
var apiServiceGroupVersions = []schema.GroupVersion{
// these are all the apigroups we manage
{Group: "apps.openshift.io", Version: "v1"},
{Group: "authorization.openshift.io", Version: "v1"},
{Group: "build.openshift.io", Version: "v1"},
{Group: "image.openshift.io", Version: "v1"},
{Group: "project.openshift.io", Version: "v1"},
{Group: "quota.openshift.io", Version: "v1"},
{Group: "route.openshift.io", Version: "v1"},
{Group: "security.openshift.io", Version: "v1"},
{Group: "template.openshift.io", Version: "v1"},
func caps2strings(caps []configv1.ClusterVersionCapability) (strings []string) {
for _, cap := range caps {
strings = append(strings, string(cap))
}
return
}

func apiServices(clusterVersionInformer configinformersconfigv1.ClusterVersionInformer) ([]apiservercontrollerapiservice.ManagedAPIService, error) {
clusterVersion, err := clusterVersionInformer.Lister().Get("version")
if err != nil {
return nil, err
}

ret := []*apiregistrationv1.APIService{}
knownCaps := sets.NewString(caps2strings(clusterVersion.Status.Capabilities.KnownCapabilities)...)
capsEnabled := sets.NewString(caps2strings(clusterVersion.Status.Capabilities.EnabledCapabilities)...)

groupEnabled := make(map[string]bool)
if !knownCaps.Has(string(configv1.ClusterVersionCapabilityBuild)) || capsEnabled.Has(string(configv1.ClusterVersionCapabilityBuild)) {
groupEnabled["build.openshift.io"] = true
}
if !knownCaps.Has(string(configv1.ClusterVersionCapabilityDeploymentConfig)) || capsEnabled.Has(string(configv1.ClusterVersionCapabilityDeploymentConfig)) {
groupEnabled["apps.openshift.io"] = true
}

ret := []apiservercontrollerapiservice.ManagedAPIService{}
for _, apiServiceGroupVersion := range apiServiceGroupVersions {
obj := &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{
Name: apiServiceGroupVersion.Version + "." + apiServiceGroupVersion.Group,
Annotations: map[string]string{
"service.alpha.openshift.io/inject-cabundle": "true",
ret = append(ret, apiservercontrollerapiservice.ManagedAPIService{
Enabled: groupEnabled[apiServiceGroupVersion.Group],
APIService: &apiregistrationv1.APIService{
ObjectMeta: metav1.ObjectMeta{
Name: apiServiceGroupVersion.Version + "." + apiServiceGroupVersion.Group,
Annotations: map[string]string{
"service.alpha.openshift.io/inject-cabundle": "true",
},
},
},
Spec: apiregistrationv1.APIServiceSpec{
Group: apiServiceGroupVersion.Group,
Version: apiServiceGroupVersion.Version,
Service: &apiregistrationv1.ServiceReference{
Namespace: operatorclient.TargetNamespace,
Name: "api",
Port: utilpointer.Int32Ptr(443),
Spec: apiregistrationv1.APIServiceSpec{
Group: apiServiceGroupVersion.Group,
Version: apiServiceGroupVersion.Version,
Service: &apiregistrationv1.ServiceReference{
Namespace: operatorclient.TargetNamespace,
Name: "api",
Port: utilpointer.Int32Ptr(443),
},
GroupPriorityMinimum: 9900,
VersionPriority: 15,
},
GroupPriorityMinimum: 9900,
VersionPriority: 15,
},
}
ret = append(ret, obj)
})
}

return ret
return ret, nil
}

func apiServicesReferences() []configv1.ObjectReference {
ret := []configv1.ObjectReference{}
for _, apiService := range apiServices() {
ret = append(ret, configv1.ObjectReference{Group: "apiregistration.k8s.io", Resource: "apiservices", Name: apiService.Spec.Version + "." + apiService.Spec.Group})
for _, apiService := range apiServiceGroupVersions {
ret = append(ret, configv1.ObjectReference{Group: "apiregistration.k8s.io", Resource: "apiservices", Name: apiService.Version + "." + apiService.Group})
}
return ret
}
52 changes: 46 additions & 6 deletions pkg/operator/workload/workload_openshiftapiserver_v311_00_sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,18 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/kubernetes"
appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/klog/v2"

openshiftapi "github.com/openshift/api"
configv1 "github.com/openshift/api/config/v1"
openshiftcontrolplanev1 "github.com/openshift/api/openshiftcontrolplane/v1"
operatorv1 "github.com/openshift/api/operator/v1"
openshiftconfigclientv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1"
configinformersconfigv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/operatorclient"
"github.com/openshift/cluster-openshift-apiserver-operator/pkg/operator/v311_00_assets"
Expand All @@ -44,6 +47,8 @@ import (

const (
imageImportCAName = "image-import-ca"
buildAPIName = "build.openshift.io"
appsAPIName = "apps.openshift.io"
)

// nodeCountFunction a function to return count of nodes
Expand All @@ -55,10 +60,11 @@ type ensureAtMostOnePodPerNodeFunc func(spec *appsv1.DeploymentSpec, component s

// OpenShiftAPIServerWorkload is a struct that holds necessary data to install OpenShiftAPIServer
type OpenShiftAPIServerWorkload struct {
operatorClient v1helpers.OperatorClient
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface
kubeClient kubernetes.Interface
operatorClient v1helpers.OperatorClient
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface
clusterVersionInformer configinformersconfigv1.ClusterVersionInformer
kubeClient kubernetes.Interface

// countNodes a function to return count of nodes on which the workload will be installed
countNodes nodeCountFunc
Expand All @@ -79,6 +85,7 @@ func NewOpenShiftAPIServerWorkload(
operatorClient v1helpers.OperatorClient,
operatorConfigClient operatorv1client.OpenShiftAPIServersGetter,
openshiftConfigClient openshiftconfigclientv1.ConfigV1Interface,
clusterVersionInformer configinformersconfigv1.ClusterVersionInformer,
countNodes nodeCountFunc,
ensureAtMostOnePodPerNode ensureAtMostOnePodPerNodeFunc,
targetNamespace string,
Expand All @@ -91,6 +98,7 @@ func NewOpenShiftAPIServerWorkload(
operatorClient: operatorClient,
operatorConfigClient: operatorConfigClient,
openshiftConfigClient: openshiftConfigClient,
clusterVersionInformer: clusterVersionInformer,
countNodes: countNodes,
ensureAtMostOnePodPerNode: ensureAtMostOnePodPerNode,
targetNamespace: targetNamespace,
Expand Down Expand Up @@ -147,7 +155,7 @@ func (c *OpenShiftAPIServerWorkload) Sync(ctx context.Context, syncContext facto
}
operatorConfig := originalOperatorConfig.DeepCopy()

_, _, err = manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx, c.kubeClient.CoreV1(), syncContext.Recorder(), operatorConfig)
_, _, err = manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx, c.kubeClient.CoreV1(), c.clusterVersionInformer, syncContext.Recorder(), operatorConfig)
if err != nil {
errors = append(errors, fmt.Errorf("%q: %v", "configmap", err))
}
Expand Down Expand Up @@ -273,15 +281,47 @@ func manageOpenShiftAPIServerImageImportCA_v311_00_to_latest(ctx context.Context
return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap)
}

func manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx context.Context, client coreclientv1.ConfigMapsGetter, recorder events.Recorder, operatorConfig *operatorv1.OpenShiftAPIServer) (*corev1.ConfigMap, bool, error) {
func caps2strings(caps []configv1.ClusterVersionCapability) (strings []string) {
for _, cap := range caps {
strings = append(strings, string(cap))
}
return
}

func manageOpenShiftAPIServerConfigMap_v311_00_to_latest(ctx context.Context, client coreclientv1.ConfigMapsGetter, clusterVersionInformer configinformersconfigv1.ClusterVersionInformer, recorder events.Recorder, operatorConfig *operatorv1.OpenShiftAPIServer) (*corev1.ConfigMap, bool, error) {
configMap := resourceread.ReadConfigMapV1OrDie(v311_00_assets.MustAsset("v3.11.0/openshift-apiserver/cm.yaml"))
defaultConfig := v311_00_assets.MustAsset("v3.11.0/config/defaultconfig.yaml")

clusterVersion, err := clusterVersionInformer.Lister().Get("version")
if err != nil {
return nil, false, err
}

knownCaps := sets.NewString(caps2strings(clusterVersion.Status.Capabilities.KnownCapabilities)...)
capsEnabled := sets.NewString(caps2strings(clusterVersion.Status.Capabilities.EnabledCapabilities)...)

apiServers := []string{}
if knownCaps.Has(string(configv1.ClusterVersionCapabilityBuild)) && !capsEnabled.Has(string(configv1.ClusterVersionCapabilityBuild)) {
klog.Infof("Capability %q not enabled, disabling 'openshift.io/build' controller", configv1.ClusterVersionCapabilityBuild)
apiServers = append(apiServers, "-openshift.io/build")
}

if knownCaps.Has(string(configv1.ClusterVersionCapabilityDeploymentConfig)) && !capsEnabled.Has(string(configv1.ClusterVersionCapabilityDeploymentConfig)) {
klog.Infof("Capability %q not enabled, disabling 'openshift.io/apps' controller", configv1.ClusterVersionCapabilityDeploymentConfig)
apiServers = append(apiServers, "-openshift.io/apps")
}

apiServers = append(apiServers, "*")

apiServersConfig := []byte("{\"apiServers\":[\"" + strings.Join(apiServers, "\", \"") + "\"]}")

requiredConfigMap, _, err := resourcemerge.MergePrunedConfigMap(
&openshiftcontrolplanev1.OpenShiftAPIServerConfig{},
configMap,
"config.yaml",
nil,
defaultConfig,
apiServersConfig,
operatorConfig.Spec.ObservedConfig.Raw,
operatorConfig.Spec.UnsupportedConfigOverrides.Raw,
)
Expand Down
12 changes: 12 additions & 0 deletions vendor/github.com/openshift/api/config/v1/types_cluster_version.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions vendor/github.com/openshift/api/openshiftcontrolplane/v1/types.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

19 changes: 13 additions & 6 deletions .../github.com/openshift/library-go/pkg/operator/apiserver/controller/apiservice/apigroup.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit d54a619

Please sign in to comment.