UPSTREAM: <carry>: kube-apiserver: allow injection of kube-apiserver …

…options

UPSTREAM: <carry>: kube-apiserver: allow rewiring

OpenShift-Rebase-Source: 56b49c9
OpenShift-Rebase-Source: bcf574c

cmd/kube-apiserver/app/options/options.go

@@ -87,6 +87,8 @@ type ServerRunOptions struct {
 	ServiceAccountTokenMaxExpiration time.Duration
 
 	ShowHiddenMetricsForVersion string
+
+	OpenShiftConfig string
 }
 
 // NewServerRunOptions creates a new ServerRunOptions object with default parameters
@@ -241,5 +243,9 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
 		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key.")
 
+	fs.StringVar(&s.OpenShiftConfig, "openshift-config", s.OpenShiftConfig, "config for openshift")
+	fs.MarkDeprecated("openshift-config", "to be removed")
+	fs.MarkHidden("openshift-config")
+
 	return fss
 }

cmd/kube-apiserver/app/patch_openshift.go

@@ -0,0 +1,33 @@
+package app
+
+import (
+	"k8s.io/apiserver/pkg/admission"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	clientgoinformers "k8s.io/client-go/informers"
+	"k8s.io/kubernetes/openshift-kube-apiserver/openshiftkubeapiserver"
+	"k8s.io/kubernetes/pkg/master"
+)
+
+var OpenShiftKubeAPIServerConfigPatch openshiftkubeapiserver.KubeAPIServerConfigFunc = nil
+
+type KubeAPIServerServerFunc func(server *master.Master) error
+
+func PatchKubeAPIServerConfig(config *genericapiserver.Config, versionedInformers clientgoinformers.SharedInformerFactory, pluginInitializers *[]admission.PluginInitializer) (genericapiserver.DelegationTarget, error) {
+	if OpenShiftKubeAPIServerConfigPatch == nil {
+		return genericapiserver.NewEmptyDelegate(), nil
+	}
+
+	return OpenShiftKubeAPIServerConfigPatch(config, versionedInformers, pluginInitializers)
+}
+
+var OpenShiftKubeAPIServerServerPatch KubeAPIServerServerFunc = nil
+
+func PatchKubeAPIServerServer(server *master.Master) error {
+	if OpenShiftKubeAPIServerServerPatch == nil {
+		return nil
+	}
+
+	return OpenShiftKubeAPIServerServerPatch(server)
+}
+
+var StartingDelegate genericapiserver.DelegationTarget = genericapiserver.NewEmptyDelegate()

cmd/kube-apiserver/app/server.go

@@ -29,6 +29,11 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/kubernetes/openshift-kube-apiserver/configdefault"
+	"k8s.io/kubernetes/openshift-kube-apiserver/enablement"
+	"k8s.io/kubernetes/openshift-kube-apiserver/openshiftkubeapiserver"
+
+	"github.com/go-openapi/spec"
 	"github.com/spf13/cobra"
 
 	oteltrace "go.opentelemetry.io/otel/trace"
@@ -120,6 +125,31 @@ cluster's shared state through which all other components interact.`,
 			}
 			cliflag.PrintFlags(fs)
 
+			if len(s.OpenShiftConfig) > 0 {
+				enablement.ForceOpenShift()
+				openshiftConfig, err := enablement.GetOpenshiftConfig(s.OpenShiftConfig)
+				if err != nil {
+					klog.Fatal(err)
+				}
+
+				// this forces a patch to be called
+				// TODO we're going to try to remove bits of the patching.
+				configPatchFn, serverPatchContext := openshiftkubeapiserver.NewOpenShiftKubeAPIServerConfigPatch(genericapiserver.NewEmptyDelegate(), openshiftConfig)
+				OpenShiftKubeAPIServerConfigPatch = configPatchFn
+				OpenShiftKubeAPIServerServerPatch = serverPatchContext.PatchServer
+
+				args, err := openshiftkubeapiserver.ConfigToFlags(openshiftConfig)
+				if err != nil {
+					return err
+				}
+				// hopefully this resets the flags?
+				if err := cmd.ParseFlags(args); err != nil {
+					return err
+				}
+
+				enablement.ForceGlobalInitializationForOpenShift(s)
+			}
+
 			// set default options
 			completedOptions, err := Complete(s)
 			if err != nil {
@@ -204,6 +234,10 @@ func CreateServerChain(completedOptions completedServerRunOptions) (*aggregatora
 		return nil, err
 	}
 
+	if err := PatchKubeAPIServerServer(kubeAPIServer); err != nil {
+		return nil, err
+	}
+
 	// aggregator comes last in the chain
 	aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, kubeAPIServerConfig.ExtraConfig.VersionedInformers, serviceResolver, kubeAPIServerConfig.ExtraConfig.ProxyTransport, pluginInitializer)
 	if err != nil {
@@ -342,6 +376,7 @@ func CreateKubeAPIServerConfig(s completedServerRunOptions) (
 func buildGenericConfig(
 	s *options.ServerRunOptions,
 	proxyTransport *http.Transport,
+
 ) (
 	genericConfig *genericapiserver.Config,
 	versionedInformers clientgoinformers.SharedInformerFactory,
@@ -461,6 +496,15 @@ func buildGenericConfig(
 		return
 	}
 
+	StartingDelegate, err = PatchKubeAPIServerConfig(genericConfig, versionedInformers, &pluginInitializers)
+	if err != nil {
+		lastErr = fmt.Errorf("failed to patch: %v", err)
+		return
+	}
+
+	if enablement.IsOpenShift() {
+		configdefault.SetAdmissionDefaults(s, versionedInformers, clientgoExternalClient)
+	}
 	err = s.Admission.ApplyTo(
 		genericConfig,
 		versionedInformers,

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/patch_policy.go

@@ -0,0 +1,65 @@
+package bootstrappolicy
+
+import (
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1"
+)
+
+var ClusterRoles = clusterRoles
+
+func OpenshiftClusterRoles() []rbacv1.ClusterRole {
+	const (
+		// These are valid under the "nodes" resource
+		NodeMetricsSubresource = "metrics"
+		NodeStatsSubresource   = "stats"
+		NodeSpecSubresource    = "spec"
+		NodeLogSubresource     = "log"
+	)
+
+	roles := clusterRoles()
+	roles = append(roles, []rbacv1.ClusterRole{
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "system:node-admin",
+			},
+			Rules: []rbacv1.PolicyRule{
+				// Allow read-only access to the API objects
+				rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("nodes").RuleOrDie(),
+				// Allow all API calls to the nodes
+				rbacv1helpers.NewRule("proxy").Groups(legacyGroup).Resources("nodes").RuleOrDie(),
+				rbacv1helpers.NewRule("*").Groups(legacyGroup).Resources("nodes/proxy", "nodes/"+NodeMetricsSubresource, "nodes/"+NodeSpecSubresource, "nodes/"+NodeStatsSubresource, "nodes/"+NodeLogSubresource).RuleOrDie(),
+			},
+		},
+		{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "system:node-reader",
+			},
+			Rules: []rbacv1.PolicyRule{
+				// Allow read-only access to the API objects
+				rbacv1helpers.NewRule(Read...).Groups(legacyGroup).Resources("nodes").RuleOrDie(),
+				// Allow read access to node metrics
+				rbacv1helpers.NewRule("get").Groups(legacyGroup).Resources("nodes/"+NodeMetricsSubresource, "nodes/"+NodeSpecSubresource).RuleOrDie(),
+				// Allow read access to stats
+				// Node stats requests are submitted as POSTs.  These creates are non-mutating
+				rbacv1helpers.NewRule("get", "create").Groups(legacyGroup).Resources("nodes/" + NodeStatsSubresource).RuleOrDie(),
+				// TODO: expose other things like /healthz on the node once we figure out non-resource URL policy across systems
+			},
+		},
+	}...)
+
+	addClusterRoleLabel(roles)
+	return roles
+}
+
+var ClusterRoleBindings = clusterRoleBindings
+
+func OpenshiftClusterRoleBindings() []rbacv1.ClusterRoleBinding {
+	bindings := clusterRoleBindings()
+	bindings = append(bindings, []rbacv1.ClusterRoleBinding{
+		rbacv1helpers.NewClusterBinding("system:node-admin").Users("system:master", "system:kube-apiserver").Groups("system:node-admins").BindingOrDie(),
+	}...)
+
+	addClusterRoleBindingLabel(bindings)
+	return bindings
+}

plugin/pkg/auth/authorizer/rbac/bootstrappolicy/policy.go

@@ -188,8 +188,8 @@ func NodeRules() []rbacv1.PolicyRule {
 	return nodePolicyRules
 }
 
-// ClusterRoles returns the cluster roles to bootstrap an API server with
-func ClusterRoles() []rbacv1.ClusterRole {
+// clusterRoles returns the cluster roles to bootstrap an API server with
+func clusterRoles() []rbacv1.ClusterRole {
 	roles := []rbacv1.ClusterRole{
 		{
 			// a "root" role which can do absolutely anything
@@ -606,7 +606,7 @@ func ClusterRoles() []rbacv1.ClusterRole {
 const systemNodeRoleName = "system:node"
 
 // ClusterRoleBindings return default rolebindings to the default roles
-func ClusterRoleBindings() []rbacv1.ClusterRoleBinding {
+func clusterRoleBindings() []rbacv1.ClusterRoleBinding {
 	rolebindings := []rbacv1.ClusterRoleBinding{
 		rbacv1helpers.NewClusterBinding("cluster-admin").Groups(user.SystemPrivilegedGroup).BindingOrDie(),
 		rbacv1helpers.NewClusterBinding("system:monitoring").Groups(user.MonitoringGroup).BindingOrDie(),