Skip to content

Commit

Permalink
UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-c…
Browse files Browse the repository at this point in the history
…ontroller-manager

UPSTREAM: <carry>: (squash) kube-controller-manager: allow running bare kube-controller-manager

UPSTREAM: <carry>: kube-controller-manager: allow running bare kube-controller-manager

openshift-rebase(v1.24):source=18bbb151dd9

openshift-rebase(v1.24):source=18bbb151dd9

openshift-rebase(v1.24):source=18bbb151dd9

UPSTREAM: <carry>: (squash) remove egressnetworkpolicies from gc ignored resources

egressnetworkpolicies should not be in garbage collector ignored
resources, so users can delete them using "--cascade=foreground" flag.

Signed-off-by: Flavio Fernandes <flaviof@redhat.com>

openshift-rebase(v1.24):source=771b4b56597
  • Loading branch information
deads2k authored and soltysh committed Aug 18, 2022
1 parent 79a9908 commit 8668eb4
Show file tree
Hide file tree
Showing 12 changed files with 843 additions and 12 deletions.
5 changes: 4 additions & 1 deletion cmd/kube-controller-manager/app/apps.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@ import (
)

func startDaemonSetController(ctx context.Context, controllerContext ControllerContext) (controller.Interface, bool, error) {
dsc, err := daemon.NewDaemonSetsController(
dsc, err := daemon.NewNodeSelectorAwareDaemonSetsController(
controllerContext.OpenShiftContext.OpenShiftDefaultProjectNodeSelector,
controllerContext.OpenShiftContext.KubeDefaultProjectNodeSelector,
controllerContext.InformerFactory.Core().V1().Namespaces(),
controllerContext.InformerFactory.Apps().V1().DaemonSets(),
controllerContext.InformerFactory.Apps().V1().ControllerRevisions(),
controllerContext.InformerFactory.Core().V1().Pods(),
Expand Down
2 changes: 2 additions & 0 deletions cmd/kube-controller-manager/app/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (

// Config is the main context object for the controller manager.
type Config struct {
OpenShiftContext OpenShiftContext

ComponentConfig kubectrlmgrconfig.KubeControllerManagerConfiguration

SecureServing *apiserver.SecureServingInfo
Expand Down
9 changes: 9 additions & 0 deletions cmd/kube-controller-manager/app/config/patch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package config

// OpenShiftContext is additional context that we need to launch the kube-controller-manager for openshift.
// Basically, this holds our additional config information.
type OpenShiftContext struct {
OpenShiftConfig string
OpenShiftDefaultProjectNodeSelector string
KubeDefaultProjectNodeSelector string
}
20 changes: 15 additions & 5 deletions cmd/kube-controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ import (
"k8s.io/controller-manager/pkg/informerfactory"
"k8s.io/controller-manager/pkg/leadermigration"
"k8s.io/klog/v2"
kubefeatures "k8s.io/kubernetes/pkg/features"

"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
Expand Down Expand Up @@ -135,6 +134,10 @@ controller, and serviceaccounts controller.`,
return err
}

if err := ShimForOpenShift(s, c); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
return err
}
return Run(c.Complete(), wait.NeverStop)
},
Args: func(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -320,6 +323,8 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {

// ControllerContext defines the context object for controller
type ControllerContext struct {
OpenShiftContext config.OpenShiftContext

// ClientBuilder will provide a client for this controller to use
ClientBuilder clientbuilder.ControllerClientBuilder

Expand Down Expand Up @@ -497,7 +502,12 @@ func GetAvailableResources(clientBuilder clientbuilder.ControllerClientBuilder)
// the shared-informers client and token controller.
func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder clientbuilder.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
var sharedInformers informers.SharedInformerFactory
if InformerFactoryOverride == nil {
sharedInformers = informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
} else {
sharedInformers = InformerFactoryOverride
}

metadataClient := metadata.NewForConfigOrDie(rootClientBuilder.ConfigOrDie("metadata-informers"))
metadataInformers := metadatainformer.NewSharedInformerFactory(metadataClient, ResyncPeriod(s)())
Expand Down Expand Up @@ -528,6 +538,7 @@ func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clien
}

ctx := ControllerContext{
OpenShiftContext: s.OpenShiftContext,
ClientBuilder: clientBuilder,
InformerFactory: sharedInformers,
ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
Expand Down Expand Up @@ -646,11 +657,10 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
controllerContext.InformerFactory.Core().V1().ServiceAccounts(),
controllerContext.InformerFactory.Core().V1().Secrets(),
c.rootClientBuilder.ClientOrDie("tokens-controller"),
serviceaccountcontroller.TokensControllerOptions{
applyOpenShiftServiceServingCertCA(serviceaccountcontroller.TokensControllerOptions{
TokenGenerator: tokenGenerator,
RootCA: rootCA,
AutoGenerate: !utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LegacyServiceAccountTokenNoAutoGeneration),
},
}),
)
if err != nil {
return nil, true, fmt.Errorf("error creating Tokens controller: %v", err)
Expand Down
9 changes: 9 additions & 0 deletions cmd/kube-controller-manager/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ type KubeControllerManagerOptions struct {
Master string
Kubeconfig string
ShowHiddenMetricsForVersion string
OpenShiftContext kubecontrollerconfig.OpenShiftContext
}

// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
Expand Down Expand Up @@ -261,6 +262,11 @@ func (s *KubeControllerManagerOptions) Flags(allControllers []string, disabledBy
fs := fss.FlagSet("misc")
fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
var dummy string
fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
fs.StringVar(&s.OpenShiftContext.OpenShiftConfig, "openshift-config", s.OpenShiftContext.OpenShiftConfig, "indicates that this process should be compatible with openshift start master")
fs.MarkHidden("openshift-config")
utilfeature.DefaultMutableFeatureGate.AddFlag(fss.FlagSet("generic"))

return fss
Expand Down Expand Up @@ -360,6 +366,9 @@ func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) e
return err
}
}

c.OpenShiftContext = s.OpenShiftContext

return nil
}

Expand Down
84 changes: 84 additions & 0 deletions cmd/kube-controller-manager/app/patch.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package app

import (
"io/ioutil"
"path"

"k8s.io/apimachinery/pkg/util/json"
kyaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/informers"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
)

var InformerFactoryOverride informers.SharedInformerFactory

func ShimForOpenShift(controllerManagerOptions *options.KubeControllerManagerOptions, controllerManager *config.Config) error {
if len(controllerManager.OpenShiftContext.OpenShiftConfig) == 0 {
return nil
}

// TODO this gets removed when no longer take flags and no longer build a recycler template
openshiftConfig, err := getOpenShiftConfig(controllerManager.OpenShiftContext.OpenShiftConfig)
if err != nil {
return err
}

// TODO this should be replaced by using a flex volume to inject service serving cert CAs into pods instead of adding it to the sa token
if err := applyOpenShiftServiceServingCertCAFunc(path.Dir(controllerManager.OpenShiftContext.OpenShiftConfig), openshiftConfig); err != nil {
return err
}

// skip GC on some openshift resources
// TODO this should be replaced by discovery information in some way
if err := applyOpenShiftGCConfig(controllerManager); err != nil {
return err
}

if err := applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions, openshiftConfig); err != nil {
return err
}

// Overwrite the informers, because we have our custom generic informers for quota.
// TODO update quota to create its own informer like garbage collection
if informers, err := newInformerFactory(controllerManager.Kubeconfig); err != nil {
return err
} else {
InformerFactoryOverride = informers
}

return nil
}

func getOpenShiftConfig(configFile string) (map[string]interface{}, error) {
configBytes, err := ioutil.ReadFile(configFile)
if err != nil {
return nil, err
}
jsonBytes, err := kyaml.ToJSON(configBytes)
if err != nil {
return nil, err
}
config := map[string]interface{}{}
if err := json.Unmarshal(jsonBytes, &config); err != nil {
return nil, err
}

return config, nil
}

func applyOpenShiftConfigDefaultProjectSelector(controllerManagerOptions *options.KubeControllerManagerOptions, openshiftConfig map[string]interface{}) error {
projectConfig, ok := openshiftConfig["projectConfig"]
if !ok {
return nil
}

castProjectConfig := projectConfig.(map[string]interface{})
defaultNodeSelector, ok := castProjectConfig["defaultNodeSelector"]
if !ok {
return nil
}
controllerManagerOptions.OpenShiftContext.OpenShiftDefaultProjectNodeSelector = defaultNodeSelector.(string)

return nil
}
39 changes: 39 additions & 0 deletions cmd/kube-controller-manager/app/patch_gc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package app

import (
gcconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"

"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
)

func applyOpenShiftGCConfig(controllerManager *config.Config) error {
// TODO make this configurable or discoverable. This is going to prevent us from running the stock GC controller
// IF YOU ADD ANYTHING TO THIS LIST, MAKE SURE THAT YOU UPDATE THEIR STRATEGIES TO PREVENT GC FINALIZERS
controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources = append(controllerManager.ComponentConfig.GarbageCollectorController.GCIgnoredResources,
// explicitly disabled from GC for now - not enough value to track them
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindingrestrictions"},
gcconfig.GroupResource{Group: "network.openshift.io", Resource: "clusternetworks"},
gcconfig.GroupResource{Group: "network.openshift.io", Resource: "hostsubnets"},
gcconfig.GroupResource{Group: "network.openshift.io", Resource: "netnamespaces"},
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclientauthorizations"},
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthclients"},
gcconfig.GroupResource{Group: "quota.openshift.io", Resource: "clusterresourcequotas"},
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "groups"},
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "identities"},
gcconfig.GroupResource{Group: "user.openshift.io", Resource: "users"},
gcconfig.GroupResource{Group: "image.openshift.io", Resource: "images"},

// virtual resource
gcconfig.GroupResource{Group: "project.openshift.io", Resource: "projects"},
// virtual and unwatchable resource, surfaced via rbac.authorization.k8s.io objects
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterroles"},
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "clusterrolebindings"},
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "roles"},
gcconfig.GroupResource{Group: "authorization.openshift.io", Resource: "rolebindings"},
// these resources contain security information in their names, and we don't need to track them
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthaccesstokens"},
gcconfig.GroupResource{Group: "oauth.openshift.io", Resource: "oauthauthorizetokens"},
)

return nil
}

0 comments on commit 8668eb4

Please sign in to comment.