Skip to content

Commit

Permalink
switch to post-starthooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@deads2k
deads2k committed Aug 3, 2017
1 parent 6f528db commit 49f4406
Show file tree
Hide file tree
Showing 9 changed files with 215 additions and 193 deletions.
5 changes: 4 additions & 1 deletion pkg/authorization/admission/restrictusers/restrictusers_test.go
View file
Expand Up @@ -567,6 +567,9 @@ func TestAdmission(t *testing.T) {
},
}

stopCh := make(chan struct{})
defer close(stopCh)

for _, tc := range testCases {
kclientset := fake.NewSimpleClientset(otestclient.UpstreamObjects(tc.objects)...)
oclient := otestclient.NewSimpleFake(otestclient.OriginObjects(tc.objects)...)
Expand All @@ -581,7 +584,7 @@ func TestAdmission(t *testing.T) {

groupCache := usercache.NewGroupCache(&groupCache{[]userapi.Group{group}})
plugin.(oadmission.WantsGroupCache).SetGroupCache(groupCache)
groupCache.Run()
groupCache.RunUntil(stopCh)

err = admission.Validate(plugin)
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion pkg/authorization/admission/restrictusers/subjectchecker_test.go
View file
Expand Up @@ -340,10 +340,13 @@ func TestSubjectCheckers(t *testing.T) {
},
}

stopCh := make(chan struct{})
defer close(stopCh)

kclient := fake.NewSimpleClientset(otestclient.UpstreamObjects(objects)...)
oclient := otestclient.NewSimpleFake(otestclient.OriginObjects(objects)...)
groupCache := usercache.NewGroupCache(&groupCache{[]userapi.Group{group}})
groupCache.Run()
groupCache.RunUntil(stopCh)
// This is a terrible, horrible, no-good, very bad hack to avoid a race
// condition between the test "allow regular user by group membership"
// and the group cache's initialisation.
Expand Down
163 changes: 19 additions & 144 deletions pkg/cmd/server/origin/ensure.go
View file
Expand Up @@ -3,17 +3,16 @@ package origin
import (
"fmt"
"io/ioutil"
"time"

"github.com/golang/glog"

kapierror "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
genericapiserver "k8s.io/apiserver/pkg/server"
kapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/rbac"
"k8s.io/kubernetes/pkg/client/retry"
kbootstrappolicy "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy"

"github.com/openshift/origin/pkg/oc/admin/policy"
Expand All @@ -23,154 +22,14 @@ import (
clusterpolicystorage "github.com/openshift/origin/pkg/authorization/registry/clusterpolicy/etcd"
"github.com/openshift/origin/pkg/cmd/server/admin"
"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
"github.com/openshift/origin/pkg/security/legacyclient"
)

// ensureOpenShiftSharedResourcesNamespace is called as part of global policy initialization to ensure shared namespace exists
func (c *MasterConfig) ensureOpenShiftSharedResourcesNamespace() {
if _, err := c.KubeClientsetInternal().Core().Namespaces().Get(c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace, metav1.GetOptions{}); kapierror.IsNotFound(err) {
namespace, createErr := c.KubeClientsetInternal().Core().Namespaces().Create(&kapi.Namespace{ObjectMeta: metav1.ObjectMeta{Name: c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace}})
if createErr != nil {
glog.Errorf("Error creating namespace: %v due to %v\n", c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace, createErr)
return
}

c.ensureNamespaceServiceAccountRoleBindings(namespace)
}
}

// ensureOpenShiftInfraNamespace is called as part of global policy initialization to ensure infra namespace exists
func (c *MasterConfig) ensureOpenShiftInfraNamespace() {
ns := c.Options.PolicyConfig.OpenShiftInfrastructureNamespace

// Ensure namespace exists
namespace, err := c.KubeClientsetInternal().Core().Namespaces().Create(&kapi.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})
if kapierror.IsAlreadyExists(err) {
// Get the persisted namespace
namespace, err = c.KubeClientsetInternal().Core().Namespaces().Get(ns, metav1.GetOptions{})
if err != nil {
glog.Errorf("Error getting namespace %s: %v", ns, err)
return
}
} else if err != nil {
glog.Errorf("Error creating namespace %s: %v", ns, err)
return
}

for _, role := range bootstrappolicy.ControllerRoles() {
reconcileRole := &policy.ReconcileClusterRolesOptions{
RolesToReconcile: []string{role.Name},
Confirmed: true,
Union: true,
Out: ioutil.Discard,
RoleClient: c.PrivilegedLoopbackOpenShiftClient.ClusterRoles(),
}
if err := reconcileRole.RunReconcileClusterRoles(nil, nil); err != nil {
glog.Errorf("Could not reconcile %v: %v\n", role.Name, err)
}
}
for _, roleBinding := range bootstrappolicy.ControllerRoleBindings() {
reconcileRoleBinding := &policy.ReconcileClusterRoleBindingsOptions{
RolesToReconcile: []string{roleBinding.RoleRef.Name},
Confirmed: true,
Union: true,
Out: ioutil.Discard,
RoleBindingClient: c.PrivilegedLoopbackOpenShiftClient.ClusterRoleBindings(),
}
if err := reconcileRoleBinding.RunReconcileClusterRoleBindings(nil, nil); err != nil {
glog.Errorf("Could not reconcile %v: %v\n", roleBinding.Name, err)
}
}

c.ensureNamespaceServiceAccountRoleBindings(namespace)
}

// ensureDefaultNamespaceServiceAccountRoles initializes roles for service accounts in the default namespace
func (c *MasterConfig) ensureDefaultNamespaceServiceAccountRoles() {
// Wait for the default namespace
var namespace *kapi.Namespace
for i := 0; i < 30; i++ {
ns, err := c.KubeClientsetInternal().Core().Namespaces().Get(metav1.NamespaceDefault, metav1.GetOptions{})
if err == nil {
namespace = ns
break
}
if kapierror.IsNotFound(err) {
time.Sleep(time.Second)
continue
}
glog.Errorf("Error adding service account roles to %q namespace: %v", metav1.NamespaceDefault, err)
return
}
if namespace == nil {
glog.Errorf("Namespace %q not found, could not initialize the %q namespace", metav1.NamespaceDefault, metav1.NamespaceDefault)
return
}

c.ensureNamespaceServiceAccountRoleBindings(namespace)
}

// ensureNamespaceServiceAccountRoleBindings initializes roles for service accounts in the namespace
func (c *MasterConfig) ensureNamespaceServiceAccountRoleBindings(namespace *kapi.Namespace) {
const ServiceAccountRolesInitializedAnnotation = "openshift.io/sa.initialized-roles"

// Short-circuit if we're already initialized
if namespace.Annotations[ServiceAccountRolesInitializedAnnotation] == "true" {
return
}

hasErrors := false
for _, binding := range bootstrappolicy.GetBootstrapServiceAccountProjectRoleBindings(namespace.Name) {
addRole := &policy.RoleModificationOptions{
RoleName: binding.RoleRef.Name,
RoleNamespace: binding.RoleRef.Namespace,
RoleBindingAccessor: policy.NewLocalRoleBindingAccessor(namespace.Name, c.ServiceAccountRoleBindingClient()),
Subjects: binding.Subjects,
}
if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { return addRole.AddRole() }); err != nil {
glog.Errorf("Could not add service accounts to the %v role in the %q namespace: %v\n", binding.RoleRef.Name, namespace.Name, err)
hasErrors = true
}
}

// If we had errors, don't register initialization so we can try again
if hasErrors {
return
}

if namespace.Annotations == nil {
namespace.Annotations = map[string]string{}
}
namespace.Annotations[ServiceAccountRolesInitializedAnnotation] = "true"
// Log any error other than a conflict (the update will be retried and recorded again on next startup in that case)
if _, err := c.KubeClientsetInternal().Core().Namespaces().Update(namespace); err != nil && !kapierror.IsConflict(err) {
glog.Errorf("Error recording adding service account roles to %q namespace: %v", namespace.Name, err)
}
}

func (c *MasterConfig) ensureDefaultSecurityContextConstraints() {
ns := c.Options.PolicyConfig.OpenShiftInfrastructureNamespace
bootstrapSCCGroups, bootstrapSCCUsers := bootstrappolicy.GetBoostrapSCCAccess(ns)

for _, scc := range bootstrappolicy.GetBootstrapSecurityContextConstraints(bootstrapSCCGroups, bootstrapSCCUsers) {
_, err := legacyclient.NewFromClient(c.KubeClientsetInternal().Core().RESTClient()).Create(&scc)
if kapierror.IsAlreadyExists(err) {
continue
}
if err != nil {
glog.Errorf("Unable to create default security context constraint %s. Got error: %v", scc.Name, err)
continue
}
glog.Infof("Created default security context constraint %s", scc.Name)
}
}

// ensureComponentAuthorizationRules initializes the cluster policies
func (c *MasterConfig) ensureComponentAuthorizationRules() {
func (c *MasterConfig) ensureComponentAuthorizationRules(context genericapiserver.PostStartHookContext) error {
clusterPolicyStorage, err := clusterpolicystorage.NewREST(c.RESTOptionsGetter)
if err != nil {
glog.Errorf("Error creating policy storage: %v", err)
return
return nil
}
clusterPolicyRegistry := clusterpolicyregistry.NewRegistry(clusterPolicyStorage)
ctx := apirequest.WithNamespace(apirequest.NewContext(), "")
Expand Down Expand Up @@ -244,4 +103,20 @@ func (c *MasterConfig) ensureComponentAuthorizationRules() {
if err := reconcileRoleBindings.RunReconcileClusterRoleBindings(nil, nil); err != nil {
glog.Errorf("Could not auto reconcile role bindings: %v\n", err)
}

return nil
}

// ensureOpenShiftSharedResourcesNamespace is called as part of global policy initialization to ensure shared namespace exists
func (c *MasterConfig) ensureOpenShiftSharedResourcesNamespace(context genericapiserver.PostStartHookContext) error {
if _, err := c.PrivilegedLoopbackKubernetesClientsetInternal.Core().Namespaces().Get(c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace, metav1.GetOptions{}); kapierror.IsNotFound(err) {
namespace, createErr := c.PrivilegedLoopbackKubernetesClientsetInternal.Core().Namespaces().Create(&kapi.Namespace{ObjectMeta: metav1.ObjectMeta{Name: c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace}})
if createErr != nil {
glog.Errorf("Error creating namespace: %v due to %v\n", c.Options.PolicyConfig.OpenShiftSharedResourcesNamespace, createErr)
return nil
}

EnsureNamespaceServiceAccountRoleBindings(c.PrivilegedLoopbackKubernetesClientsetInternal, c.PrivilegedLoopbackOpenShiftClient, namespace)
}
return nil
}
24 changes: 9 additions & 15 deletions pkg/cmd/server/origin/master.go
View file
Expand Up @@ -225,6 +225,15 @@ func (c *MasterConfig) Run(kubeAPIServerConfig *kubeapiserver.Config, assetConfi
return err
}

// add post-start hooks
aggregatedAPIServer.GenericAPIServer.AddPostStartHook("user.openshift.io-groupcache",
func(context apiserver.PostStartHookContext) error {
c.GroupCache.RunUntil(context.StopCh)
return nil
})
aggregatedAPIServer.GenericAPIServer.AddPostStartHook("template.openshift.io-sharednamespace", c.ensureOpenShiftSharedResourcesNamespace)
aggregatedAPIServer.GenericAPIServer.AddPostStartHook("authorization.openshift.io-bootstrapclusterroles", c.ensureComponentAuthorizationRules)

go aggregatedAPIServer.GenericAPIServer.PrepareRun().Run(stopCh)

// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
Expand Down Expand Up @@ -309,21 +318,6 @@ func (c *MasterConfig) buildHandlerChain(assetConfig *AssetConfig) (func(http.Ha
}, nil
}

// InitializeObjects ensures objects in Kubernetes and etcd are properly populated.
// Requires a Kube client to be established and that etcd be started.
func (c *MasterConfig) InitializeObjects() {
// Create required policy rules if needed
c.ensureComponentAuthorizationRules()
// Ensure the default SCCs are created
c.ensureDefaultSecurityContextConstraints()
// Bind default roles for service accounts in the default namespace if needed
c.ensureDefaultNamespaceServiceAccountRoles()
// Create the infra namespace
c.ensureOpenShiftInfraNamespace()
// Create the shared resource namespace
c.ensureOpenShiftSharedResourcesNamespace()
}

// getRequestContextMapper returns a mapper from requests to contexts, initializing it if needed
func (c *MasterConfig) getRequestContextMapper() apirequest.RequestContextMapper {
if c.RequestContextMapper == nil {
Expand Down

0 comments on commit 49f4406

Please sign in to comment.