Merge pull request kubernetes#16529 from alvaroaleman/boskos-retry-on…

…-conflict Boskos: Retry on conflict
adshmh · Mar 2, 2020 · bd87a79 · bd87a79
2 parents c921a88 + 6b0ac1e
commit bd87a79
Show file tree

Hide file tree

Showing 6 changed files with 451 additions and 285 deletions.
diff --git a/boskos/handlers/BUILD.bazel b/boskos/handlers/BUILD.bazel
@@ -27,8 +27,11 @@ go_test(
         "//boskos/crds:go_default_library",
         "//boskos/ranch:go_default_library",
         "@com_github_go_test_deep//:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
         "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
         "@io_k8s_sigs_controller_runtime//pkg/client/fake:go_default_library",
     ],
 )

diff --git a/boskos/handlers/handlers_test.go b/boskos/handlers/handlers_test.go
@@ -18,7 +18,9 @@ package handlers
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -29,8 +31,11 @@ import (
 	"testing"
 	"time"
 
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
 	"k8s.io/test-infra/boskos/client"
@@ -61,7 +66,8 @@ func MakeTestRanch(resources []runtime.Object) *ranch.Ranch {
 	for _, obj := range resources {
 		obj.(metav1.Object).SetNamespace(ns)
 	}
-	s := ranch.NewTestingStorage(fakectrlruntimeclient.NewFakeClient(resources...), ns, func() time.Time { return fakeNow })
+	client := &onceConflictingClient{Client: fakectrlruntimeclient.NewFakeClient(resources...)}
+	s := ranch.NewTestingStorage(client, ns, func() time.Time { return fakeNow })
 	r, _ := ranch.NewRanch("", s, testTTL)
 	return r
 }
@@ -856,3 +862,18 @@ func compareWithFixture(testName string, actualData []byte) error {
 
 	return nil
 }
+
+// onceConflictingClient returns an IsConflict error on the first Update request it receives. It
+// is used to verify that there is retrying for conflicts in place.
+type onceConflictingClient struct {
+	didConflict bool
+	ctrlruntimeclient.Client
+}
+
+func (occ *onceConflictingClient) Update(ctx context.Context, obj runtime.Object, opts ...ctrlruntimeclient.UpdateOption) error {
+	if !occ.didConflict {
+		occ.didConflict = true
+		return kerrors.NewConflict(schema.GroupResource{}, "obj", errors.New("conflicting as requested"))
+	}
+	return occ.Client.Update(ctx, obj, opts...)
+}
diff --git a/boskos/ranch/BUILD.bazel b/boskos/ranch/BUILD.bazel
@@ -19,8 +19,12 @@ go_test(
         "@com_github_go_test_deep//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@io_k8s_apimachinery//pkg/api/equality:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
         "@io_k8s_apimachinery//pkg/runtime:go_default_library",
+        "@io_k8s_apimachinery//pkg/runtime/schema:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/errors:go_default_library",
+        "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
         "@io_k8s_sigs_controller_runtime//pkg/client/fake:go_default_library",
     ],
 )
@@ -36,11 +40,14 @@ go_library(
     deps = [
         "//boskos/common:go_default_library",
         "//boskos/crds:go_default_library",
-        "@com_github_hashicorp_go_multierror//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@io_k8s_apimachinery//pkg/api/errors:go_default_library",
         "@io_k8s_apimachinery//pkg/apis/meta/v1:go_default_library",
         "@io_k8s_apimachinery//pkg/types:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/errors:go_default_library",
         "@io_k8s_apimachinery//pkg/util/sets:go_default_library",
+        "@io_k8s_apimachinery//pkg/util/wait:go_default_library",
+        "@io_k8s_client_go//util/retry:go_default_library",
         "@io_k8s_sigs_controller_runtime//pkg/client:go_default_library",
     ],
 )