Merge pull request #15138 from zetaab/exitgracefully

exit nodeup gracefully if server already exists in k8s

cmd/kops-controller/pkg/server/server.go

@@ -142,6 +142,12 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 
 	id, err := s.verifier.VerifyToken(ctx, r, r.Header.Get("Authorization"), body, s.opt.Server.UseInstanceIDForNodeName)
 	if err != nil {
+		// means that we should exit nodeup gracefully
+		if err == bootstrap.ErrAlreadyExists {
+			w.WriteHeader(http.StatusConflict)
+			klog.Infof("%s: %v", r.RemoteAddr, err)
+			return
+		}
 		klog.Infof("bootstrap %s verify err: %v", r.RemoteAddr, err)
 		w.WriteHeader(http.StatusForbidden)
 		// don't return the error; this allows us to have richer errors without security implications

pkg/bootstrap/authenticate.go

@@ -18,9 +18,12 @@ package bootstrap
 
 import (
 	"context"
+	"errors"
 	"net/http"
 )
 
+var ErrAlreadyExists = errors.New("node already exists")
+
 // Authenticator generates authentication credentials for requests.
 type Authenticator interface {
 	CreateToken(body []byte) (string, error)

pkg/kopscontrollerclient/client.go

@@ -27,6 +27,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"time"
 
@@ -150,6 +151,12 @@ func (b *Client) Query(ctx context.Context, req any, resp any) error {
 		defer response.Body.Close()
 	}
 
+	// if we receive StatusConflict it means that we should exit gracefully
+	if response.StatusCode == http.StatusConflict {
+		klog.Infof("kops-controller returned status code %d", response.StatusCode)
+		os.Exit(0)
+	}
+
 	if response.StatusCode != http.StatusOK {
 		detail := ""
 		if response.Body != nil {

upup/pkg/fi/cloudup/openstack/verifier.go

@@ -154,7 +154,7 @@ func (o openstackVerifier) VerifyToken(ctx context.Context, rawRequest *http.Req
 	// check from kubernetes API does the instance already exist
 	_, err = o.kubeClient.CoreV1().Nodes().Get(ctx, instance.Name, v1.GetOptions{})
 	if err == nil {
-		return nil, fmt.Errorf("server %q is already joined to kubernetes cluster", instance.Name)
+		return nil, bootstrap.ErrAlreadyExists
 	}
 	if err != nil && !errors.IsNotFound(err) {
 		return nil, fmt.Errorf("got error while querying kubernetes api: %w", err)