Skip to content

Commit

Permalink
deploy webhook by yaml
Browse files Browse the repository at this point in the history 
Signed-off-by: hwdef <hwdefcom@outlook.com>
  • Loading branch information
@hwdef
hwdef committed Jul 11, 2022
1 parent 0b52e8f commit 5614179
Show file tree
Hide file tree
Showing 6 changed files with 896 additions and 121 deletions.
8 changes: 5 additions & 3 deletions cmd/webhook-manager/app/server.go
View file
Expand Up @@ -77,11 +77,13 @@ func Run(config *options.Config) error {

klog.V(3).Infof("Registered '%s' as webhook.", service.Path)
http.HandleFunc(service.Path, service.Handler)

klog.V(3).Infof("Registered configuration for webhook <%s>", service.Path)
registerWebhookConfig(kubeClient, config, service, config.CaCertData)
})

if err = addCaCertForWebhook(kubeClient, config.CaCertData); err != nil {
return fmt.Errorf("failed to add caCert for webhook %v", err)
}
klog.V(3).Infof("Successfully added caCert for all webhooks")

webhookServeError := make(chan struct{})
stopChannel := make(chan os.Signal, 1)
signal.Notify(stopChannel, syscall.SIGTERM, syscall.SIGINT)
Expand Down
183 changes: 65 additions & 118 deletions cmd/webhook-manager/app/util.go
View file
Expand Up @@ -17,93 +17,95 @@ limitations under the License.
package app

import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"regexp"
"strings"

"fmt"
v1 "k8s.io/api/admissionregistration/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/klog"

"volcano.sh/apis/pkg/client/clientset/versioned"
"volcano.sh/volcano/cmd/webhook-manager/app/options"
"volcano.sh/volcano/pkg/webhooks/router"
)

func registerWebhookConfig(kubeClient *kubernetes.Clientset, config *options.Config, service *router.AdmissionService, caBundle []byte) {
sideEffect := v1.SideEffectClassNoneOnDryRun
reviewVersions := []string{"v1"}
webhookLabelSelector := &metav1.LabelSelector{}
clientConfig := v1.WebhookClientConfig{
CABundle: caBundle,
}
if config.WebhookURL != "" {
url := config.WebhookURL + service.Path
clientConfig.URL = &url
klog.Infof("The URL of webhook manager is <%s>.", url)
}
if config.WebhookName != "" && config.WebhookNamespace != "" {
clientConfig.Service = &v1.ServiceReference{
Name: config.WebhookName,
Namespace: config.WebhookNamespace,
Path: &service.Path,
}
klog.Infof("The service of webhook manager is <%s/%s/%s>.",
config.WebhookName, config.WebhookNamespace, service.Path)
var (
validatingWebhooksName = []string{
"volcano-admission-service-jobs-validate",
"volcano-admission-service-pods-validate",
"volcano-admission-service-queues-validate",
}
if config.IgnoredNamespaces != "" {
ignoredNamespaces := strings.Split(strings.TrimSpace(config.IgnoredNamespaces), ",")
klog.Infof("The ignored namespaces list of webhook manager is <%v>.",
ignoredNamespaces)
webhookLabelSelector = &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Values: ignoredNamespaces,
Operator: "NotIn",
Key: "kubernetes.io/metadata.name",
},
},
}
mutatingWebhooksName = []string{
"volcano-admission-service-pods-mutate",
"volcano-admission-service-queues-mutate",
"volcano-admission-service-podgroups-mutate",
"volcano-admission-service-jobs-mutate",
}
if service.MutatingConfig != nil {
for i := range service.MutatingConfig.Webhooks {
service.MutatingConfig.Webhooks[i].SideEffects = &sideEffect
service.MutatingConfig.Webhooks[i].AdmissionReviewVersions = reviewVersions
service.MutatingConfig.Webhooks[i].ClientConfig = clientConfig
service.MutatingConfig.Webhooks[i].NamespaceSelector = webhookLabelSelector
}
)

service.MutatingConfig.ObjectMeta.Name = webhookConfigName(config.WebhookName, service.Path)
func addCaCertForWebhook(kubeClient *kubernetes.Clientset, caBundle []byte) error {
for _, mutatingWebhookName := range mutatingWebhooksName {
var mutatingWebhook *v1.MutatingWebhookConfiguration
if err := wait.PollInfinite(time.Second, func() (done bool, err error) {
mutatingWebhook, err = kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Get(context.TODO(), mutatingWebhookName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
klog.Errorln(err)
return false, nil
}
return false, fmt.Errorf("failed to get mutating webhook %v", err)
}
return true, nil
}); err != nil {
return fmt.Errorf("failed to get mutating webhook %v", err)
}

if err := registerMutateWebhook(kubeClient, service.MutatingConfig); err != nil {
klog.Errorf("Failed to register mutating admission webhook (%s): %v",
service.Path, err)
} else {
klog.V(3).Infof("Registered mutating webhook for path <%s>.", service.Path)
for index := 0; index < len(mutatingWebhook.Webhooks); index++ {
if mutatingWebhook.Webhooks[index].ClientConfig.CABundle == nil ||
bytes.Equal(mutatingWebhook.Webhooks[index].ClientConfig.CABundle, caBundle) {
mutatingWebhook.Webhooks[index].ClientConfig.CABundle = caBundle
}
}
}
if service.ValidatingConfig != nil {
for i := range service.ValidatingConfig.Webhooks {
service.ValidatingConfig.Webhooks[i].SideEffects = &sideEffect
service.ValidatingConfig.Webhooks[i].AdmissionReviewVersions = reviewVersions
service.ValidatingConfig.Webhooks[i].ClientConfig = clientConfig
service.ValidatingConfig.Webhooks[i].NamespaceSelector = webhookLabelSelector
if _, err := kubeClient.AdmissionregistrationV1().MutatingWebhookConfigurations().Update(context.TODO(), mutatingWebhook, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("failed to update mutating admission webhooks %v %v", mutatingWebhookName, err)
}
}

service.ValidatingConfig.ObjectMeta.Name = webhookConfigName(config.WebhookName, service.Path)
for _, validatingWebhookName := range validatingWebhooksName {
var validatingWebhook *v1.ValidatingWebhookConfiguration
if err := wait.PollInfinite(time.Second, func() (done bool, err error) {
validatingWebhook, err = kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Get(context.TODO(), validatingWebhookName, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
klog.Errorln(err)
return false, nil
}
return false, fmt.Errorf("failed to get validating webhook %v", err)
}
return true, nil
}); err != nil {
return fmt.Errorf("failed to get validating webhook %v", err)
}

if err := registerValidateWebhook(kubeClient, service.ValidatingConfig); err != nil {
klog.Errorf("Failed to register validating admission webhook (%s): %v",
service.Path, err)
} else {
klog.V(3).Infof("Registered validating webhook for path <%s>.", service.Path)
for index := 0; index < len(validatingWebhook.Webhooks); index++ {
if validatingWebhook.Webhooks[index].ClientConfig.CABundle == nil ||
bytes.Equal(validatingWebhook.Webhooks[index].ClientConfig.CABundle, caBundle) {
validatingWebhook.Webhooks[index].ClientConfig.CABundle = caBundle
}
}
if _, err := kubeClient.AdmissionregistrationV1().ValidatingWebhookConfigurations().Update(context.TODO(), validatingWebhook, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("failed to update validating admission webhooks %v %v", validatingWebhookName, err)
}
}

return nil
}

// getKubeClient Get a clientset with restConfig.
Expand Down Expand Up @@ -164,58 +166,3 @@ func configTLS(config *options.Config, restConfig *rest.Config) *tls.Config {
klog.Fatal("tls: failed to find any tls config data")
return &tls.Config{}
}

func registerMutateWebhook(clientset *kubernetes.Clientset, hook *v1.MutatingWebhookConfiguration) error {
client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations()
existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
if err == nil && existing != nil {
klog.V(4).Infof("Updating MutatingWebhookConfiguration %v", hook)
existing.Webhooks = hook.Webhooks
if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil {
return err
}
} else {
klog.V(4).Infof("Creating MutatingWebhookConfiguration %v", hook)
if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil {
return err
}
}

return nil
}

func registerValidateWebhook(clientset *kubernetes.Clientset, hook *v1.ValidatingWebhookConfiguration) error {
client := clientset.AdmissionregistrationV1().ValidatingWebhookConfigurations()

existing, err := client.Get(context.TODO(), hook.Name, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
return err
}
if err == nil && existing != nil {
existing.Webhooks = hook.Webhooks
klog.V(4).Infof("Updating ValidatingWebhookConfiguration %v", hook)
if _, err := client.Update(context.TODO(), existing, metav1.UpdateOptions{}); err != nil {
return err
}
} else {
klog.V(4).Infof("Creating ValidatingWebhookConfiguration %v", hook)
if _, err := client.Create(context.TODO(), hook, metav1.CreateOptions{}); err != nil {
return err
}
}

return nil
}

func webhookConfigName(name, path string) string {
if name == "" {
name = "webhook"
}

re := regexp.MustCompile(`-+`)
raw := strings.Join([]string{name, strings.ReplaceAll(path, "/", "-")}, "-")
return re.ReplaceAllString(raw, "-")
}
1 change: 1 addition & 0 deletions hack/generate-yaml.sh
View file
Expand Up @@ -118,6 +118,7 @@ ${HELM_BIN_DIR}/helm template ${VK_ROOT}/installer/helm/chart/volcano --namespac
-s templates/scheduling_v1beta1_podgroup.yaml \
-s templates/scheduling_v1beta1_queue.yaml \
-s templates/nodeinfo_v1alpha1_numatopologies.yaml \
-s templates/webhooks.yaml \
>> ${DEPLOYMENT_FILE}

${HELM_BIN_DIR}/helm template ${VK_ROOT}/installer/helm/chart/volcano --namespace volcano-monitoring \
Expand Down

0 comments on commit 5614179

Please sign in to comment.