prevent OOM on stream error

cockroachdb · Oct 6, 2018 · 039524b · 039524b
1 parent bab5d13
commit 039524b
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 6 deletions.
diff --git a/request-cert/k8s_certificates.go b/request-cert/k8s_certificates.go
@@ -37,9 +37,10 @@ const (
 )
 
 var (
-	kubeConfig  = flag.String("kubeconfig", "", "config file if running from outside the cluster")
-	client      *kubernetes.Clientset
-	clientError error
+	kubeConfig   = flag.String("kubeconfig", "", "config file if running from outside the cluster")
+	client       *kubernetes.Clientset
+	clientError  error
+	ChannelError = errors.New("error on the channel")
 )
 
 func getClient() (*kubernetes.Clientset, error) {
@@ -65,7 +66,7 @@ func initClient() (*kubernetes.Clientset, error) {
 	return c, err
 }
 
-func getKubernetesCertificate(csrName string, csr []byte, wantServerAuth bool) ([]byte, error) {
+func getKubernetesCertificate(csrName string, csr []byte, wantServerAuth bool, allowPrevious bool) ([]byte, error) {
 	client, err := getClient()
 	if err != nil {
 		return nil, err
@@ -92,6 +93,12 @@ func getKubernetesCertificate(csrName string, csr []byte, wantServerAuth bool) (
 
 	fmt.Printf("Sending create request: %s for %s\n", req.Name, *addresses)
 	resp, err := client.Certificates().CertificateSigningRequests().Create(req)
+
+	if err != nil && k8s_errors.IsAlreadyExists(err) && allowPrevious {
+		fmt.Printf("Attempting to use previous CSR: %s\n", req.Name)
+		getOpts := types.GetOptions{TypeMeta: types.TypeMeta{Kind: "CertificateSigningRequest"}}
+		resp, err = client.Certificates().CertificateSigningRequests().Get(req.Name, getOpts)
+	}
 	if err != nil {
 		return nil, errors.Wrapf(err, "CertificateSigningRequest.Create(%s) failed", req.Name)
 	}
@@ -117,7 +124,7 @@ func getKubernetesCertificate(csrName string, csr []byte, wantServerAuth bool) (
 		select {
 		case event, ok := <-watchCh:
 			if !ok {
-				break
+				return nil, ChannelError
 			}
 
 			if event.Object.(*certificates.CertificateSigningRequest).UID != resp.UID {

diff --git a/request-cert/main.go b/request-cert/main.go
@@ -150,7 +150,10 @@ func requestCertificate(
 	)
 
 	// Send CSR for approval and certificate generation.
-	pemCert, err := getKubernetesCertificate(csrName, pemCSR, wantServerAuth)
+	pemCert, err := getKubernetesCertificate(csrName, pemCSR, wantServerAuth, false)
+	for i := 0; i < 10 && err == ChannelError; i++ {
+		pemCert, err = getKubernetesCertificate(csrName, pemCSR, wantServerAuth, true)
+	}
 	if err != nil {
 		return nil, nil, err
 	}