diff --git a/go.mod b/go.mod index c2692accc..526789b3c 100644 --- a/go.mod +++ b/go.mod @@ -126,16 +126,16 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/automaxprocs v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.43.0 // indirect + golang.org/x/crypto v0.45.0 // indirect golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect - golang.org/x/net v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/term v0.36.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.37.0 // indirect + golang.org/x/tools v0.38.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index b8decfaf9..03aed3514 100644 --- a/go.sum +++ b/go.sum @@ -343,6 +343,8 @@ golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= @@ -372,6 +374,8 @@ golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= @@ -389,6 +393,8 @@ golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -407,6 +413,8 @@ golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -414,6 +422,8 @@ golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -424,6 +434,8 @@ golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -440,6 +452,8 @@ golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/splunk/enterprise/clustermanager_test.go b/pkg/splunk/enterprise/clustermanager_test.go index 92eb22e7e..c30640abe 100644 --- a/pkg/splunk/enterprise/clustermanager_test.go +++ b/pkg/splunk/enterprise/clustermanager_test.go @@ -563,6 +563,7 @@ func TestApplyClusterManagerWithSmartstore(t *testing.T) { {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermanager-smartstore"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-manager"}, {MetaName: "*v1.Pod-test-splunk-stack1-cluster-manager-0"}, {MetaName: "*v1.StatefulSet-test-splunk-test-monitoring-console"}, {MetaName: "*v4.ClusterManager-test-stack1"}, diff --git a/pkg/splunk/enterprise/clustermaster_test.go b/pkg/splunk/enterprise/clustermaster_test.go index 2439a4249..fea79b90d 100644 --- a/pkg/splunk/enterprise/clustermaster_test.go +++ b/pkg/splunk/enterprise/clustermaster_test.go @@ -263,6 +263,7 @@ func TestApplyClusterMasterWithSmartstore(t *testing.T) { {MetaName: "*v1.ConfigMap-test-splunk-stack1-clustermaster-smartstore"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"}, {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1-cluster-master"}, {MetaName: "*v1.Pod-test-splunk-stack1-cluster-master-0"}, {MetaName: "*v1.StatefulSet-test-splunk-test-monitoring-console"}, {MetaName: "*v3.ClusterMaster-test-stack1"}, diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index 8513972e0..92f562c5a 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -769,7 +769,7 @@ func TestIndexerClusterPodManager(t *testing.T) { listmockCall := []spltest.MockFuncCall{ {ListOpts: listOpts}} - wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} + wantCalls := map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} // test 1 ready pod mockHandlers := []spltest.MockHTTPHandler{ @@ -822,6 +822,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1"}, {MetaName: "*v1.Pod-test-splunk-stack1-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-indexer-0"}, @@ -836,6 +837,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1"}, {MetaName: "*v1.Pod-test-splunk-stack1-0"}, } mockHandlers = []spltest.MockHTTPHandler{mockHandlers[0], mockHandlers[1]} @@ -862,7 +864,7 @@ func TestIndexerClusterPodManager(t *testing.T) { statefulSet.Status.Replicas = 2 statefulSet.Status.ReadyReplicas = 2 statefulSet.Status.UpdatedReplicas = 2 - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[4], funcCalls[4], funcCalls[0]}, "Create": {funcCalls[1]}} method = "indexerClusterPodManager.Update(Pod Not Found)" indexerClusterPodManagerUpdateTester(t, method, mockHandlers, 1, enterpriseApi.PhaseScalingDown, statefulSet, wantCalls, nil, statefulSet, pod) @@ -885,6 +887,7 @@ func TestIndexerClusterPodManager(t *testing.T) { {MetaName: "*v1.Secret-test-splunk-test-secret"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1"}, {MetaName: "*v1.Pod-test-splunk-manager1-cluster-manager-0"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"}, diff --git a/pkg/splunk/enterprise/searchheadcluster_test.go b/pkg/splunk/enterprise/searchheadcluster_test.go index 68118bf67..569d0be8a 100644 --- a/pkg/splunk/enterprise/searchheadcluster_test.go +++ b/pkg/splunk/enterprise/searchheadcluster_test.go @@ -317,7 +317,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { }, } method = "searchHeadClusterPodManager.Update(All pods ready)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "List": {listmockCall[0]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseReady, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => transition to detention @@ -337,7 +337,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { ) pod.ObjectMeta.Labels["controller-revision-hash"] = "v0" method = "searchHeadClusterPodManager.Update(Quarantine Pod)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5], funcCalls[2], funcCalls[2]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => wait for searches to drain @@ -345,13 +345,13 @@ func TestSearchHeadClusterPodManager(t *testing.T) { mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"status":"Up"`, `"status":"ManualDetention"`, 1) mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":0`, `"active_historical_search_count":1`, 1) method = "searchHeadClusterPodManager.Update(Draining Searches)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod needs update => delete pod mockHandlers[0].Body = strings.Replace(mockHandlers[0].Body, `"active_historical_search_count":1`, `"active_historical_search_count":0`, 1) method = "searchHeadClusterPodManager.Update(Delete Pod)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5]}, "Create": {funcCalls[1]}, "Delete": {funcCalls[5]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test pod update finished => release from detention @@ -364,7 +364,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { Body: ``, }) method = "searchHeadClusterPodManager.Update(Release Quarantine)" - wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}} + wantCalls = map[string][]spltest.MockFuncCall{"Get": {funcCalls[0], funcCalls[1], funcCalls[1], funcCalls[2], funcCalls[2], funcCalls[0], funcCalls[5], funcCalls[2]}, "Create": {funcCalls[1]}} searchHeadClusterPodManagerTester(t, method, mockHandlers, 1, enterpriseApi.PhaseUpdating, statefulSet, wantCalls, nil, statefulSet, pod) // test scale down => remove member @@ -394,6 +394,7 @@ func TestSearchHeadClusterPodManager(t *testing.T) { {MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-0"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"}, + {MetaName: "*v1.StatefulSet-test-splunk-stack1"}, {MetaName: "*v1.Pod-test-splunk-stack1-search-head-1"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-etc-splunk-stack1-1"}, {MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"}, diff --git a/pkg/splunk/splkcontroller/statefulset.go b/pkg/splunk/splkcontroller/statefulset.go index 2c8e2804a..3028efbdd 100644 --- a/pkg/splunk/splkcontroller/statefulset.go +++ b/pkg/splunk/splkcontroller/statefulset.go @@ -118,6 +118,17 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st "name", statefulSet.GetObjectMeta().GetName(), "namespace", statefulSet.GetObjectMeta().GetNamespace()) + // Re-fetch the StatefulSet to ensure we have the latest status, especially UpdateRevision. + // This addresses a race condition where the StatefulSet controller may not have updated + // Status.UpdateRevision yet after a spec change was applied. Without this re-fetch, + // we might incorrectly report PhaseReady when pods actually need to be recycled. + namespacedName := types.NamespacedName{Namespace: statefulSet.GetNamespace(), Name: statefulSet.GetName()} + err := c.Get(ctx, namespacedName, statefulSet) + if err != nil { + scopedLog.Error(err, "Unable to re-fetch StatefulSet for latest status") + return enterpriseApi.PhaseError, err + } + // wait for all replicas ready replicas := *statefulSet.Spec.Replicas readyReplicas := statefulSet.Status.ReadyReplicas @@ -249,7 +260,7 @@ func UpdateStatefulSetPods(ctx context.Context, c splcommon.ControllerClient, st } // Remove unwanted owner references - err := splutil.RemoveUnwantedSecrets(ctx, c, statefulSet.GetName(), statefulSet.GetNamespace()) + err = splutil.RemoveUnwantedSecrets(ctx, c, statefulSet.GetName(), statefulSet.GetNamespace()) if err != nil { return enterpriseApi.PhaseReady, err } diff --git a/pkg/splunk/splkcontroller/statefulset_test.go b/pkg/splunk/splkcontroller/statefulset_test.go index bd9b605a4..da38a38df 100644 --- a/pkg/splunk/splkcontroller/statefulset_test.go +++ b/pkg/splunk/splkcontroller/statefulset_test.go @@ -246,6 +246,8 @@ func TestUpdateStatefulSetPods(t *testing.T) { }, } statefulSet.Status.ReadyReplicas = 3 + // Add statefulSet to mock client so UpdateStatefulSetPods can re-fetch it + c.AddObject(statefulSet) c.InduceErrorKind[splcommon.MockClientInduceErrorUpdate] = rerr _, err = UpdateStatefulSetPods(ctx, c, statefulSet, &mgr, 1) if err == nil { diff --git a/pkg/splunk/test/controller.go b/pkg/splunk/test/controller.go index aa0dfb4b5..6e5871cc4 100644 --- a/pkg/splunk/test/controller.go +++ b/pkg/splunk/test/controller.go @@ -794,7 +794,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM // test scale up (zero ready so far; wait for ready) revised = current.DeepCopy() current.Status.ReadyReplicas = 0 - scaleUpCalls := map[string][]MockFuncCall{"Get": {funcCalls[0]}} + scaleUpCalls := map[string][]MockFuncCall{"Get": {funcCalls[0], funcCalls[0]}} methodPlus = fmt.Sprintf("%s(%s)", method, "ScalingUp, 0 ready") PodManagerUpdateTester(t, methodPlus, mgr, 1, enterpriseApi.PhasePending, revised, scaleUpCalls, nil, current) @@ -809,7 +809,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM replicas = 1 current.Status.Replicas = 1 current.Status.ReadyReplicas = 1 - updateCalls = map[string][]MockFuncCall{"Get": {funcCalls[0]}, "Update": {funcCalls[0]}} + updateCalls = map[string][]MockFuncCall{"Get": {funcCalls[0], funcCalls[0]}, "Update": {funcCalls[0]}} methodPlus = fmt.Sprintf("%s(%s)", method, "ScalingUp, Update Replicas 1=>2") PodManagerUpdateTester(t, methodPlus, mgr, 2, enterpriseApi.PhaseScalingUp, revised, updateCalls, nil, current, pod) @@ -827,7 +827,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM {MetaName: "*v1.PersistentVolumeClaim-test-pvc-var-splunk-stack1-1"}, } scaleDownCalls := map[string][]MockFuncCall{ - "Get": {funcCalls[0], pvcCalls[0], pvcCalls[1]}, + "Get": {funcCalls[0], funcCalls[0], pvcCalls[0], pvcCalls[1]}, "Update": {funcCalls[0]}, "Delete": pvcCalls, } @@ -845,7 +845,7 @@ func PodManagerTester(t *testing.T, method string, mgr splcommon.StatefulSetPodM replicas = 1 current.Status.Replicas = 1 current.Status.ReadyReplicas = 1 - podCalls := []MockFuncCall{funcCalls[0], {MetaName: "*v1.Pod-test-splunk-stack1-0"}} + podCalls := []MockFuncCall{funcCalls[0], funcCalls[0], {MetaName: "*v1.Pod-test-splunk-stack1-0"}} getPodCalls := map[string][]MockFuncCall{"Get": podCalls} //getPodCalls := map[string][]MockFuncCall{} methodPlus = fmt.Sprintf("%s(%s)", method, "Pod not found") diff --git a/test/appframework_aws/c3/manager_appframework_test.go b/test/appframework_aws/c3/manager_appframework_test.go index e00da4428..826c48d93 100644 --- a/test/appframework_aws/c3/manager_appframework_test.go +++ b/test/appframework_aws/c3/manager_appframework_test.go @@ -363,9 +363,6 @@ var _ = Describe("c3appfw test", func() { // Ensure Cluster Manager goes to Ready phase testenv.ClusterManagerReady(ctx, deployment, testcaseEnvInst) - // Ensure Monitoring Console goes to Ready phase - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - // // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) @@ -439,15 +436,14 @@ var _ = Describe("c3appfw test", func() { // Wait for License Manager to be in READY phase testenv.LicenseManagerReady(ctx, deployment, testcaseEnvInst) - // // Verify Monitoring Console is ready and stays in ready state - testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) - // // Ensure Search Head Cluster go to Ready phase testenv.SearchHeadClusterReady(ctx, deployment, testcaseEnvInst) // // Ensure Indexers go to Ready phase testenv.SingleSiteIndexersReady(ctx, deployment, testcaseEnvInst) + // // Verify Monitoring Console is ready and stays in ready state + testenv.VerifyMonitoringConsoleReady(ctx, deployment, deployment.GetName(), mc, testcaseEnvInst) }) })