Skip to content

Commit

Permalink
Merge pull request openshift#1536 from benluddy/runtime-config
Browse files Browse the repository at this point in the history
Set runtime-config in lockstep with feature-gates, if needed.
  • Loading branch information
openshift-merge-robot committed Aug 15, 2023
2 parents 7d5e2e0 + 24ccf54 commit 7864ef2
Show file tree
Hide file tree
Showing 7 changed files with 316 additions and 14 deletions.
1 change: 1 addition & 0 deletions bindata/assets/config/defaultconfig.yaml
Expand Up @@ -75,6 +75,7 @@ apiServerArguments:
- StorageObjectInUseProtection
- TaintNodesByCondition
- ValidatingAdmissionWebhook
- ValidatingAdmissionPolicy
- authorization.openshift.io/RestrictSubjectBindings
- authorization.openshift.io/ValidateRoleBindingRestriction
- config.openshift.io/DenyDeleteClusterConfiguration
Expand Down
2 changes: 2 additions & 0 deletions bindata/bootkube/config/bootstrap-config-overrides.yaml
Expand Up @@ -63,6 +63,8 @@ apiServerArguments:
- /etc/kubernetes/secrets/apiserver-proxy.key
requestheader-client-ca-file:
- /etc/kubernetes/secrets/aggregator-signer.crt
runtime-config: {{range .RuntimeConfig}}
- {{.}}{{end}}
service-account-key-file:
- /etc/kubernetes/secrets/service-account.pub
- /etc/kubernetes/secrets/bound-service-account-signing-key.pub
Expand Down
21 changes: 20 additions & 1 deletion pkg/cmd/render/render.go
Expand Up @@ -9,7 +9,6 @@ import (
"encoding/pem"
"errors"
"fmt"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"io/ioutil"
"net"
"os"
Expand All @@ -19,14 +18,17 @@ import (
configv1 "github.com/openshift/api/config/v1"
kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
"github.com/openshift/cluster-kube-apiserver-operator/bindata"
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/apienablement"
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/auth"
libgoaudit "github.com/openshift/library-go/pkg/operator/apiserver/audit"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
genericrender "github.com/openshift/library-go/pkg/operator/render"
genericrenderoptions "github.com/openshift/library-go/pkg/operator/render/options"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
kyaml "k8s.io/apimachinery/pkg/util/yaml"
auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
"k8s.io/klog/v2"
Expand All @@ -43,10 +45,16 @@ type renderOpts struct {
clusterConfigFile string
clusterAuthFile string
infraConfigFile string

groupVersionsByFeatureGate map[configv1.FeatureGateName][]schema.GroupVersion
}

// NewRenderCommand creates a render command.
func NewRenderCommand() *cobra.Command {
return newRenderCommand()
}

func newRenderCommand(testOverrides ...func(*renderOpts)) *cobra.Command {
renderOpts := renderOpts{
generic: *genericrenderoptions.NewGenericOptions(),
manifest: *genericrenderoptions.NewManifestOptions("kube-apiserver", "openshift/origin-hyperkube:latest"),
Expand All @@ -55,6 +63,9 @@ func NewRenderCommand() *cobra.Command {
etcdServerURLs: []string{"https://127.0.0.1:2379"},
etcdServingCA: "root-ca.crt",
}
for _, f := range testOverrides {
f(&renderOpts)
}
cmd := &cobra.Command{
Use: "render",
Short: "Render kubernetes API server bootstrap manifests, secrets and configMaps",
Expand Down Expand Up @@ -126,6 +137,9 @@ func (r *renderOpts) Complete() error {
if err := r.generic.Complete(); err != nil {
return err
}
if r.groupVersionsByFeatureGate == nil {
r.groupVersionsByFeatureGate = apienablement.DefaultGroupVersionsByFeatureGate
}
return nil
}

Expand All @@ -148,6 +162,9 @@ type TemplateData struct {
// FeatureGates is list of featuregates to apply
FeatureGates []string

// RuntimeConfig is a list of API group-versions to enable or disable.
RuntimeConfig []string

// ServiceClusterIPRange is the IP range for service IPs.
ServiceCIDR []string

Expand Down Expand Up @@ -191,6 +208,8 @@ func (r *renderOpts) Run() error {
return err
}

renderConfig.RuntimeConfig = apienablement.RuntimeConfigFromFeatureGates(featureGates, r.groupVersionsByFeatureGate)

if len(r.clusterConfigFile) > 0 {
clusterConfigFileData, err := ioutil.ReadFile(r.clusterConfigFile)
if err != nil {
Expand Down
44 changes: 33 additions & 11 deletions pkg/cmd/render/render_test.go
Expand Up @@ -13,15 +13,14 @@ import (

configv1 "github.com/openshift/api/config/v1"
kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/configobservercontroller"
libgoaudit "github.com/openshift/library-go/pkg/operator/apiserver/audit"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
genericrenderoptions "github.com/openshift/library-go/pkg/operator/render/options"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/runtime/schema"
kyaml "k8s.io/apimachinery/pkg/util/yaml"
)

Expand Down Expand Up @@ -229,17 +228,13 @@ func TestRenderCommand(t *testing.T) {
}
templateDir := filepath.Join("..", "..", "..", "bindata", "bootkube")

tempDisabledFeatureGates := configobservercontroller.FeatureBlacklist
if tempDisabledFeatureGates == nil {
tempDisabledFeatureGates = sets.New[configv1.FeatureGateName]()
}

defaultFGDir := filepath.Join("testdata", "rendered", "default-fg")

tests := []struct {
// note the name is used as a name for a temporary directory
name string
args []string
overrides []func(*renderOpts)
setupFunction func() error
testFunction func(cfg *kubecontrolplanev1.KubeAPIServerConfig) error
podTestFunction func(cfg *corev1.Pod) error
Expand All @@ -254,6 +249,13 @@ func TestRenderCommand(t *testing.T) {
"--payload-version=test",
"--rendered-manifest-files=" + defaultFGDir,
},
overrides: []func(*renderOpts){
func(opts *renderOpts) {
opts.groupVersionsByFeatureGate = map[configv1.FeatureGateName][]schema.GroupVersion{
"Foo": {{Group: "foos.example.com", Version: "v4alpha7"}},
}
},
},
testFunction: func(cfg *kubecontrolplanev1.KubeAPIServerConfig) error {
actualGates, ok := cfg.APIServerArguments["feature-gates"]
if !ok {
Expand All @@ -276,6 +278,21 @@ func TestRenderCommand(t *testing.T) {
return fmt.Errorf("%q not found on the list of expected feature gates %v", actualGate, expectedGates)
}
}

actualRuntimeConfig, ok := cfg.APIServerArguments["runtime-config"]
if !ok {
return fmt.Errorf(`missing expected "runtime-config" entry in APIServerArguments`)
}
expectedRuntimeConfig := []string{"foos.example.com/v4alpha7=true"}
if len(expectedRuntimeConfig) != len(actualRuntimeConfig) {
return fmt.Errorf("expected runtime-config of len %d, got: %v (len %d)", len(expectedRuntimeConfig), actualRuntimeConfig, len(actualRuntimeConfig))
}
for i := 0; i < len(expectedRuntimeConfig); i++ {
if expectedRuntimeConfig[i] != actualRuntimeConfig[i] {
return fmt.Errorf("expected %dth runtime-config entry %q, got %q", i+1, expectedRuntimeConfig[i], actualRuntimeConfig[i])
}
}

return nil
},
},
Expand Down Expand Up @@ -595,7 +612,7 @@ spec:
}

test.args = setOutputFlags(test.args, outputDir)
err = runRender(test.args...)
err = runRender(test.args, test.overrides)
if err != nil {
t.Fatalf("%s: got unexpected error %v", test.name, err)
}
Expand Down Expand Up @@ -698,9 +715,14 @@ func setOutputFlags(args []string, dir string) []string {
return newArgs
}

func runRender(args ...string) error {
c := NewRenderCommand()
os.Args = append([]string{""}, args...)
func runRender(args []string, overrides []func(*renderOpts)) error {
defaultTestOverrides := []func(*renderOpts){
func(opts *renderOpts) {
opts.groupVersionsByFeatureGate = map[configv1.FeatureGateName][]schema.GroupVersion{}
},
}
c := newRenderCommand(append(defaultTestOverrides, overrides...)...)
c.SetArgs(args)
return c.Execute()
}

Expand Down
@@ -0,0 +1,86 @@
package apienablement

import (
"fmt"
"sort"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"

configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/library-go/pkg/operator/configobserver"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/library-go/pkg/operator/events"
)

var DefaultGroupVersionsByFeatureGate = map[configv1.FeatureGateName][]schema.GroupVersion{
"ValidatingAdmissionPolicy": {{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}},
}

var (
featureGatesPath = []string{"apiServerArguments", "feature-gates"}
runtimeConfigPath = []string{"apiServerArguments", "runtime-config"}
)

// NewFeatureGateObserverWithRuntimeConfig returns a config observation function that observes
// feature gates and sets the --feature-gates and --runtime-config options accordingly. Since a
// mismatch between these two options can result in an unstable config, the observed value for
// either will only be set if both can be successfully set. Otherwise, the existing config is
// returned pruned but otherwise unmodified.
func NewFeatureGateObserverWithRuntimeConfig(featureWhitelist sets.Set[configv1.FeatureGateName], featureBlacklist sets.Set[configv1.FeatureGateName], featureGateAccessor featuregates.FeatureGateAccess, groupVersionsByFeatureGate map[configv1.FeatureGateName][]schema.GroupVersion) configobserver.ObserveConfigFunc {

featureGateObserver := featuregates.NewObserveFeatureFlagsFunc(
featureWhitelist,
featureBlacklist,
featureGatesPath,
featureGateAccessor,
)

return newFeatureGateObserverWithRuntimeConfig(featureGateObserver, featureGateAccessor, groupVersionsByFeatureGate)
}

func newFeatureGateObserverWithRuntimeConfig(featureGateObserver configobserver.ObserveConfigFunc, featureGateAccessor featuregates.FeatureGateAccess, groupVersionsByFeatureGate map[configv1.FeatureGateName][]schema.GroupVersion) configobserver.ObserveConfigFunc {
return func(listers configobserver.Listers, recorder events.Recorder, existingConfig map[string]interface{}) (observedConfig map[string]interface{}, errs []error) {
defer func() {
observedConfig = configobserver.Pruned(observedConfig, featureGatesPath, runtimeConfigPath)
}()

if !featureGateAccessor.AreInitialFeatureGatesObserved() {
return existingConfig, nil
}

featureGates, err := featureGateAccessor.CurrentFeatureGates()
if err != nil {
return existingConfig, []error{err}
}

observedConfig, errs = featureGateObserver(listers, recorder, existingConfig)

runtimeConfig := RuntimeConfigFromFeatureGates(featureGates, groupVersionsByFeatureGate)
if len(runtimeConfig) == 0 {
return observedConfig, errs
}

if err := unstructured.SetNestedStringSlice(observedConfig, runtimeConfig, runtimeConfigPath...); err != nil {
// The new feature gate config is broken without its required APIs.
return existingConfig, append(errs, err)
}

return observedConfig, errs
}
}

func RuntimeConfigFromFeatureGates(featureGates featuregates.FeatureGate, groupVersionsByFeatureGate map[configv1.FeatureGateName][]schema.GroupVersion) []string {
var entries []string
for name, gvs := range groupVersionsByFeatureGate {
if !featureGates.Enabled(name) {
continue
}
for _, gv := range gvs {
entries = append(entries, fmt.Sprintf("%s=true", gv.String()))
}
}
sort.Strings(entries)
return entries
}

0 comments on commit 7864ef2

Please sign in to comment.