Skip to content

Commit

Permalink
Clean up job for old resources (#15312)
Browse files Browse the repository at this point in the history
* clean up for old resources

* fix header

* fix job yaml writing

* lint

* updates

* new line
  • Loading branch information
skonto committed Jun 19, 2024
1 parent 408b284 commit 5a92452
Show file tree
Hide file tree
Showing 4 changed files with 174 additions and 1 deletion.
4 changes: 4 additions & 0 deletions config/core/200-roles/clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,7 @@ rules:
- apiGroups: ["acme.cert-manager.io"]
resources: ["challenges"]
verbs: ["get", "list", "create", "update", "delete", "patch", "watch"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["clusterroles"]
verbs: ["delete"]
resourceNames: ["knative-serving-certmanager"]
65 changes: 65 additions & 0 deletions config/post-install/cleanup.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
# Copyright 2024 The Knative Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: batch/v1
kind: Job
metadata:
generateName: cleanup-serving-
namespace: knative-serving
labels:
app: cleanup-serving
app.kubernetes.io/name: knative-serving
app.kubernetes.io/component: cleanup-job
app.kubernetes.io/version: devel
spec:
ttlSecondsAfterFinished: 600
backoffLimit: 10
template:
metadata:
annotations:
sidecar.istio.io/inject: "false"
labels:
app: cleanup-migration-serving
app.kubernetes.io/name: knative-serving
app.kubernetes.io/component: cleanup-job
app.kubernetes.io/version: devel
spec:
serviceAccountName: controller
restartPolicy: OnFailure
containers:
- name: cleanup
# This is the Go import path for the binary that is containerized
# and substituted here.
image: ko://knative.dev/serving/pkg/cleanup/cmd/cleanup
resources:
requests:
cpu: 100m
memory: 100Mi
limits:
cpu: 1000m
memory: 1000Mi
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsNonRoot: true
capabilities:
drop:
- ALL
seccompProfile:
type: RuntimeDefault
env:
- name: SYSTEM_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
5 changes: 4 additions & 1 deletion hack/generate-yamls.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,12 @@ readonly SERVING_DEFAULT_DOMAIN_YAML=${YAML_OUTPUT_DIR}/serving-default-domain.y
readonly SERVING_STORAGE_VERSION_MIGRATE_YAML=${YAML_OUTPUT_DIR}/serving-storage-version-migration.yaml
readonly SERVING_HPA_YAML=${YAML_OUTPUT_DIR}/serving-hpa.yaml
readonly SERVING_CRD_YAML=${YAML_OUTPUT_DIR}/serving-crds.yaml
readonly SERVING_CLEAN_UP_YAML=${YAML_OUTPUT_DIR}/cleanup.yaml
readonly SERVING_POST_INSTALL_JOBS_YAML=${YAML_OUTPUT_DIR}/serving-post-install-jobs.yaml

declare -A CONSOLIDATED_ARTIFACTS
CONSOLIDATED_ARTIFACTS=(
["${SERVING_POST_INSTALL_JOBS_YAML}"]="${SERVING_STORAGE_VERSION_MIGRATE_YAML}"
["${SERVING_POST_INSTALL_JOBS_YAML}"]="${SERVING_STORAGE_VERSION_MIGRATE_YAML} ${SERVING_CLEAN_UP_YAML}"
)
readonly CONSOLIDATED_ARTIFACTS

Expand Down Expand Up @@ -91,6 +92,8 @@ ko resolve ${KO_YAML_FLAGS} -f config/post-install/default-domain.yaml | "${LABE

ko resolve ${KO_YAML_FLAGS} -f config/post-install/storage-version-migration.yaml | "${LABEL_YAML_CMD[@]}" > "${SERVING_STORAGE_VERSION_MIGRATE_YAML}"

ko resolve ${KO_YAML_FLAGS} -f config/post-install/cleanup.yaml | "${LABEL_YAML_CMD[@]}" > "${SERVING_CLEAN_UP_YAML}"

# These don't have images, but ko will concatenate them for us.
ko resolve ${KO_YAML_FLAGS} -f config/core/300-resources/ -f config/core/300-imagecache.yaml | "${LABEL_YAML_CMD[@]}" > "${SERVING_CRD_YAML}"

Expand Down
101 changes: 101 additions & 0 deletions pkg/cleanup/cmd/cleanup/cleanup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2024 The Knative Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"context"
"flag"
"log"

"go.uber.org/zap"
apierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

"knative.dev/pkg/environment"
"knative.dev/pkg/logging"
"knative.dev/pkg/system"
)

func main() {
logger := setupLogger()
defer logger.Sync()

env := environment.ClientConfig{}
env.InitFlags(flag.CommandLine)

flag.Parse()

config, err := env.GetRESTConfig()
if err != nil {
logger.Fatalf("failed to get kubeconfig %s", err)
}

client := kubernetes.NewForConfigOrDie(config)

logger.Info("Deleting old Serving resources if any")

for _, dep := range []string{"domain-mapping", "domainmapping-webhook", "net-certmanager-controller", "net-certmanager-webhook"} {
if err = client.AppsV1().Deployments(system.Namespace()).Delete(context.Background(), dep, metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete deployment ", dep, ": ", err)
}
}

// Delete the rest of the domain mapping resources
if err = client.CoreV1().Services(system.Namespace()).Delete(context.Background(), "domainmapping-webhook", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete service domainmapping-webhook: ", err)
}
if err = client.CoreV1().Secrets(system.Namespace()).Delete(context.Background(), "domainmapping-webhook-certs", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete secret domainmapping-webhook-certs: ", err)
}
if err = client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), "webhook.domainmapping.serving.knative.dev", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete mutating webhook configuration webhook.domainmapping.serving.knative.dev: ", err)
}
if err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.Background(), "validation.webhook.domainmapping.serving.knative.dev", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete validating webhook configuration validation.webhook.domainmapping.serving.knative.dev: ", err)
}

// Delete the rest of the net-certmanager resources
if err = client.CoreV1().Services(system.Namespace()).Delete(context.Background(), "net-certmanager-controller", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete service net-certmanager-controller: ", err)
}
if err = client.CoreV1().Services(system.Namespace()).Delete(context.Background(), "net-certmanager-webhook", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete service net-certmanager-webhook: ", err)
}
if err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.Background(), "config.webhook.net-certmanager.networking.internal.knative.dev", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete validating webhook config.webhook.net-certmanager.networking.internal.knative.dev: ", err)
}
if err = client.CoreV1().Secrets(system.Namespace()).Delete(context.Background(), "net-certmanager-webhook-certs", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete secret net-certmanager-webhook-certs: ", err)
}
if err = client.RbacV1().ClusterRoles().Delete(context.Background(), "knative-serving-certmanager", metav1.DeleteOptions{}); err != nil && !apierrs.IsNotFound(err) {
logger.Fatal("failed to delete clusterrole knative-serving-certmanager: ", err)
}
logger.Info("Old Serving resource deletion completed successfully")
}

func setupLogger() *zap.SugaredLogger {
const component = "old-resource-cleanup"

config, err := logging.NewConfigFromMap(nil)
if err != nil {
log.Fatal("Failed to create logging config: ", err)
}

logger, _ := logging.NewLoggerFromConfig(config, component)
return logger
}

0 comments on commit 5a92452

Please sign in to comment.