diff --git a/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml b/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml index 0a58e8c4b..6b3c86853 100644 --- a/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_clustermasters_crd.yaml @@ -38,6 +38,9 @@ spec: spec: description: ClusterMasterSpec defines the desired state of ClusterMaster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml b/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml index 1bd149904..afdf30ddf 100644 --- a/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_indexerclusters_crd.yaml @@ -60,6 +60,9 @@ spec: description: IndexerClusterSpec defines the desired state of a Splunk Enterprise indexer cluster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml b/deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml index 0b28c6311..fb8b98dca 100644 --- a/deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_licensemasters_crd.yaml @@ -43,6 +43,9 @@ spec: description: LicenseMasterSpec defines the desired state of a Splunk Enterprise license master. properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml b/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml index 03fb918ab..fbef5798b 100644 --- a/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_searchheadclusters_crd.yaml @@ -60,6 +60,9 @@ spec: description: SearchHeadClusterSpec defines the desired state of a Splunk Enterprise search head cluster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/crds/enterprise.splunk.com_standalones_crd.yaml b/deploy/crds/enterprise.splunk.com_standalones_crd.yaml index 69f973fe0..c6e66585c 100644 --- a/deploy/crds/enterprise.splunk.com_standalones_crd.yaml +++ b/deploy/crds/enterprise.splunk.com_standalones_crd.yaml @@ -53,6 +53,9 @@ spec: description: StandaloneSpec defines the desired state of a Splunk Enterprise standalone instances. properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_clustermasters_crd.yaml b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_clustermasters_crd.yaml index 0a58e8c4b..6b3c86853 100644 --- a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_clustermasters_crd.yaml +++ b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_clustermasters_crd.yaml @@ -38,6 +38,9 @@ spec: spec: description: ClusterMasterSpec defines the desired state of ClusterMaster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_indexerclusters_crd.yaml b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_indexerclusters_crd.yaml index 1bd149904..afdf30ddf 100644 --- a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_indexerclusters_crd.yaml +++ b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_indexerclusters_crd.yaml @@ -60,6 +60,9 @@ spec: description: IndexerClusterSpec defines the desired state of a Splunk Enterprise indexer cluster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_licensemasters_crd.yaml b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_licensemasters_crd.yaml index 0b28c6311..fb8b98dca 100644 --- a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_licensemasters_crd.yaml +++ b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_licensemasters_crd.yaml @@ -43,6 +43,9 @@ spec: description: LicenseMasterSpec defines the desired state of a Splunk Enterprise license master. properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_searchheadclusters_crd.yaml b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_searchheadclusters_crd.yaml index 03fb918ab..fbef5798b 100644 --- a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_searchheadclusters_crd.yaml +++ b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_searchheadclusters_crd.yaml @@ -60,6 +60,9 @@ spec: description: SearchHeadClusterSpec defines the desired state of a Splunk Enterprise search head cluster properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_standalones_crd.yaml b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_standalones_crd.yaml index 69f973fe0..c6e66585c 100644 --- a/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_standalones_crd.yaml +++ b/deploy/olm-catalog/splunk/0.2.0/enterprise.splunk.com_standalones_crd.yaml @@ -53,6 +53,9 @@ spec: description: StandaloneSpec defines the desired state of a Splunk Enterprise standalone instances. properties: + Mock: + description: Mock to differentiate between UTs and actual reconcile + type: boolean affinity: description: Kubernetes Affinity rules that control how pods are assigned to particular nodes. diff --git a/pkg/apis/enterprise/v1alpha3/common_types.go b/pkg/apis/enterprise/v1alpha3/common_types.go index 988c68c7f..130853d08 100644 --- a/pkg/apis/enterprise/v1alpha3/common_types.go +++ b/pkg/apis/enterprise/v1alpha3/common_types.go @@ -66,6 +66,9 @@ type CommonSplunkSpec struct { // ClusterMasterRef refers to a Splunk Enterprise indexer cluster managed by the operator within Kubernetes ClusterMasterRef corev1.ObjectReference `json:"clusterMasterRef"` + + // Mock to differentiate between UTs and actual reconcile + Mock bool `json:"Mock"` } // SmartStoreSpec defines Splunk indexes and remote storage volume configuration diff --git a/pkg/splunk/client/enterprise.go b/pkg/splunk/client/enterprise.go index b5cbdfd1d..374970576 100644 --- a/pkg/splunk/client/enterprise.go +++ b/pkg/splunk/client/enterprise.go @@ -21,6 +21,8 @@ import ( "io/ioutil" "net/http" "regexp" + "strconv" + "strings" "time" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -66,14 +68,22 @@ func NewSplunkClient(managementURI, username, password string) *SplunkClient { } // Do processes a Splunk REST API request and unmarshals response into obj, if not nil. -func (c *SplunkClient) Do(request *http.Request, expectedStatus int, obj interface{}) error { +func (c *SplunkClient) Do(request *http.Request, expectedStatus []int, obj interface{}) error { // send HTTP response and check status request.SetBasicAuth(c.Username, c.Password) response, err := c.Client.Do(request) if err != nil { return err } - if response.StatusCode != expectedStatus { + //default set flag to false and the check response code + expectedStatusFlag := false + for i := 0; i < len(expectedStatus); i++ { + if expectedStatus[i] == response.StatusCode { + expectedStatusFlag = true + break + } + } + if expectedStatusFlag == false { return fmt.Errorf("Response code=%d from %s; want %d", response.StatusCode, request.URL, expectedStatus) } if obj == nil { @@ -95,7 +105,8 @@ func (c *SplunkClient) Get(path string, obj interface{}) error { if err != nil { return err } - return c.Do(request, 200, obj) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, obj) } // SearchHeadCaptainInfo represents the status of the search head cluster. @@ -291,7 +302,8 @@ func (c *SplunkClient) SetSearchHeadDetention(detain bool) error { if err != nil { return err } - return c.Do(request, 200, nil) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, nil) } // RemoveSearchHeadClusterMember removes a search head cluster member. @@ -595,7 +607,8 @@ func (c *SplunkClient) RemoveIndexerClusterPeer(id string) error { if err != nil { return err } - return c.Do(request, 200, nil) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, nil) } // DecommissionIndexerClusterPeer takes an indexer cluster peer offline using the decommission endpoint. @@ -611,7 +624,282 @@ func (c *SplunkClient) DecommissionIndexerClusterPeer(enforceCounts bool) error if err != nil { return err } - return c.Do(request, 200, nil) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, nil) +} + +//MCServerRolesInfo is the struct for the server roles of the localhost, in this case SplunkMonitoringConsole +type MCServerRolesInfo struct { + ServerRoles []string `json:"server_roles"` +} + +//MCDistributedPeers is the struct for information about distributed peers of the monitoring console +type MCDistributedPeers struct { + ClusterLabel []string `json:"cluster_label"` + ServerRoles []string `json:"server_roles"` +} + +//AutomateMCApplyChanges change the state of new indexers from "New" to "Configured" and add them in monitoring console asset table +func (c *SplunkClient) AutomateMCApplyChanges(mock bool) error { + if mock { + return nil + } + var configuredPeers, indexerMemberList, licenseMasterMemberList string + apiResponseServerRoles, err := c.GetMonitoringconsoleServerRoles() + if err != nil { + return err + } + + apiResponseMCDistributedPeers := struct { + Entry []struct { + Name string `json:"name"` + Content MCDistributedPeers `json:"content"` + } `json:"entry"` + }{} + path := "/services/search/distributed/peers" + err = c.Get(path, &apiResponseMCDistributedPeers) + if err != nil { + return err + } + + for _, e := range apiResponseMCDistributedPeers.Entry { + if configuredPeers == "" { + configuredPeers = e.Name + } else { + str := []string{configuredPeers, e.Name} + configuredPeers = strings.Join(str, ",") + } + for _, s := range e.Content.ServerRoles { + if s == "indexer" { + indexerMemberList = indexerMemberList + "&member=" + e.Name + } + if s == "license_master" { + licenseMasterMemberList = licenseMasterMemberList + "&member=" + e.Name + } + } + } + + for _, e := range apiResponseServerRoles.ServerRoles { + if e == "indexer" { + indexerMemberList = "&member=localhost:localhost" + indexerMemberList + } + if e == "license_master" { + licenseMasterMemberList = licenseMasterMemberList + "&member=localhost:localhost" + } + } + reqBodyIndexer := indexerMemberList + "&default=true" + reqBodyLicenseMaster := licenseMasterMemberList + "&default=false" + err = c.UpdateDMCGroups("dmc_group_indexer", reqBodyIndexer) + if err != nil { + return err + } + err = c.UpdateDMCGroups("dmc_group_license_master", reqBodyLicenseMaster) + if err != nil { + return err + } + + clusterRoleDict := make(map[string][]string) + //map of Name to Roles + for _, e := range apiResponseMCDistributedPeers.Entry { + for _, s := range e.Content.ClusterLabel { + clusterRoleDict[e.Name] = append(clusterRoleDict[e.Name], s) + } + } + //TODO: check different labels here + clusterRoleDictToDict := make(map[string][]string) + for key, value := range clusterRoleDict { + for _, val := range value { + clusterRoleDictToDict[val] = append(clusterRoleDictToDict[val], key) + } + } + + clusterRoleDictToDictString := make(map[string]string) + for key, value := range clusterRoleDictToDict { + for _, val := range value { + clusterRoleDictToDictString[key] = clusterRoleDictToDictString[key] + "&member=" + val + } + } + + for key, value := range clusterRoleDictToDictString { + if key == "" { + break + } else { + err = c.UpdateDMCClusteringLabelGroup(key, value) + if err != nil { + return err + } + } + } + apiResponseMCAssetTableBuild, err := c.GetMonitoringconsoleAssetTable() + if err != nil { + return err + } + err = c.PostMonitoringConsoleAssetTable(apiResponseMCAssetTableBuild) + if err != nil { + return err + } + UISettingsObject, err := c.GetMonitoringConsoleUISettings() + if err != nil { + return err + } + err = c.UpdateLookupUISettings(configuredPeers, UISettingsObject) + if err != nil { + return err + } + err = c.UpdateMonitoringConsoleApp() + if err != nil { + return err + } + return err +} + +//GetMonitoringconsoleServerRoles to retrive server roles of the local host or SplunkMonitoringConsole +func (c *SplunkClient) GetMonitoringconsoleServerRoles() (*MCServerRolesInfo, error) { + apiResponseServerRoles := struct { + Entry []struct { + Content MCServerRolesInfo `json:"content"` + } `json:"entry"` + }{} + path := "/services/server/info/server-info" + err := c.Get(path, &apiResponseServerRoles) + if err != nil { + return nil, err + } + if len(apiResponseServerRoles.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.ManagementURI, path) + } + return &apiResponseServerRoles.Entry[0].Content, nil +} + +//UpdateDMCGroups dmc* groups with new members +func (c *SplunkClient) UpdateDMCGroups(dmcGroupName string, groupMembers string) error { + endpoint := fmt.Sprintf("%s/services/search/distributed/groups/%s/edit", c.ManagementURI, dmcGroupName) + request, err := http.NewRequest("POST", endpoint, strings.NewReader(groupMembers)) + expectedStatus := []int{200, 201, 409} + err = c.Do(request, expectedStatus, nil) + return err +} + +//UpdateDMCClusteringLabelGroup update respective clustering group +func (c *SplunkClient) UpdateDMCClusteringLabelGroup(groupName string, groupMembers string) error { + endpoint := fmt.Sprintf("%s/services/search/distributed/groups/dmc_indexerclustergroup_%s/edit", c.ManagementURI, groupName) + reqBodyClusterGroup := groupMembers + "&default=false" + request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyClusterGroup)) + expectedStatus := []int{200, 201, 409} + err = c.Do(request, expectedStatus, nil) + return err +} + +//MCAssetBuildTable is the struct for information about asset table +type MCAssetBuildTable struct { + DispatchAutoCancel string `json:"dispatch.auto_cancel"` + DispatchBuckets int64 `json:"dispatch.buckets"` +} + +//GetMonitoringconsoleAssetTable to GET monitoring console asset table data. +func (c *SplunkClient) GetMonitoringconsoleAssetTable() (*MCAssetBuildTable, error) { + apiResponseMCAssetTableBuild := struct { + Entry []struct { + Content MCAssetBuildTable `json:"content"` + } `json:"entry"` + }{} + path := "/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full" + err := c.Get(path, &apiResponseMCAssetTableBuild) + if err != nil { + return nil, err + } + if len(apiResponseMCAssetTableBuild.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.ManagementURI, path) + } + return &apiResponseMCAssetTableBuild.Entry[0].Content, nil +} + +//PostMonitoringConsoleAssetTable to build monitoring console asset table. Kicks off the search [Build Asset Table full] +func (c *SplunkClient) PostMonitoringConsoleAssetTable(apiResponseMCAssetTableBuild *MCAssetBuildTable) error { + reqBodyAssetTable := "&trigger_actions=true&dispatch.auto_cancel=" + apiResponseMCAssetTableBuild.DispatchAutoCancel + "&dispatch.buckets=" + strconv.FormatInt(apiResponseMCAssetTableBuild.DispatchBuckets, 10) + "&dispatch.enablePreview=true" + endpoint := fmt.Sprintf("%s", c.ManagementURI) + "/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full/dispatch" + request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyAssetTable)) + request.Header.Set("Content-Type", "application/x-www-form-urlencoded") + expectedStatus := []int{200, 201, 409} + err = c.Do(request, expectedStatus, nil) + return err +} + +//UISettings is the struct for storing monitoring console app UI settings +type UISettings struct { + EaiData string `json:"eai:data"` + Disabled bool `json:"disabled"` + EaiACL string `json:"eai:acl"` + EaiAppName string `json:"eai:appName"` + EaiUserName string `json:"eai:userName"` +} + +//GetMonitoringConsoleUISettings do a Get for app UI settings +func (c *SplunkClient) GetMonitoringConsoleUISettings() (*UISettings, error) { + apiResponseUISettings := struct { + Entry []struct { + Content UISettings `json:"content"` + } `json:"entry"` + }{} + path := "/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed" + err := c.Get(path, &apiResponseUISettings) + if err != nil { + return nil, err + } + if len(apiResponseUISettings.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.ManagementURI, path) + } + return &apiResponseUISettings.Entry[0].Content, nil +} + +//UpdateLookupUISettings updates assets.csv +func (c *SplunkClient) UpdateLookupUISettings(configuredPeers string, apiResponseUISettings *UISettings) error { + reqBodyMCLookups := "configuredPeers=" + configuredPeers + "&eai:appName=" + apiResponseUISettings.EaiAppName + "&eai:acl=" + apiResponseUISettings.EaiACL + "&eai:userName=" + apiResponseUISettings.EaiUserName + "&disabled=" + strconv.FormatBool(apiResponseUISettings.Disabled) + endpoint := fmt.Sprintf("%s/servicesNS/nobody/splunk_monitoring_console/configs/conf-splunk_monitoring_console_assets/settings", c.ManagementURI) + request, err := http.NewRequest("POST", endpoint, strings.NewReader(reqBodyMCLookups)) + request.Header.Set("Content-Type", "application/x-www-form-urlencoded") + expectedStatus := []int{200, 201, 409} + err = c.Do(request, expectedStatus, nil) + return err +} + +//UpdateMonitoringConsoleApp updates the monitoring console app +func (c *SplunkClient) UpdateMonitoringConsoleApp() error { + endpoint := fmt.Sprintf("%s/servicesNS/nobody/system/apps/local/splunk_monitoring_console", c.ManagementURI) + request, err := http.NewRequest("POST", endpoint, nil) + if err != nil { + return err + } + expectedStatus := []int{200, 201} + err = c.Do(request, expectedStatus, nil) + return err +} + +//ClusterInfo is the struct for checking ClusterInfo +type ClusterInfo struct { + MultiSite string `json:"multisite"` +} + +// GetClusterInfo queries the cluster about multi-site or single-site. +//See https://docs.splunk.com/Documentation/Splunk/8.0.6/RESTREF/RESTcluster#cluster.2Fconfig +func (c *SplunkClient) GetClusterInfo(mockCall bool) (*ClusterInfo, error) { + if mockCall { + return nil, nil + } + apiResponse := struct { + Entry []struct { + Content ClusterInfo `json:"content"` + } `json:"entry"` + }{} + path := "/services/cluster/config" + err := c.Get(path, &apiResponse) + if err != nil { + return nil, err + } + if len(apiResponse.Entry) < 1 { + return nil, fmt.Errorf("Invalid response from %s%s", c.ManagementURI, path) + } + return &apiResponse.Entry[0].Content, nil } // SetIdxcSecret sets idxc_secret for a Splunk Instance @@ -624,8 +912,8 @@ func (c *SplunkClient) SetIdxcSecret(idxcSecret string) error { return err } request.Header.Set("Content-Type", "application/x-www-form-urlencoded") - - return c.Do(request, 200, nil) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, nil) } // RestartSplunk restarts specific Splunk instance @@ -637,5 +925,6 @@ func (c *SplunkClient) RestartSplunk() error { if err != nil { return err } - return c.Do(request, 200, nil) + expectedStatus := []int{200} + return c.Do(request, expectedStatus, nil) } diff --git a/pkg/splunk/client/enterprise_test.go b/pkg/splunk/client/enterprise_test.go index 9a4754482..4f8487a1b 100644 --- a/pkg/splunk/client/enterprise_test.go +++ b/pkg/splunk/client/enterprise_test.go @@ -17,6 +17,7 @@ package client import ( "fmt" "net/http" + "strings" "testing" spltest "github.com/splunk/splunk-operator/pkg/splunk/test" @@ -34,6 +35,20 @@ func splunkClientTester(t *testing.T, testMethod string, status int, body string mockSplunkClient.CheckRequests(t, testMethod) } +func splunkClientMultipleRequestTester(t *testing.T, testMethod string, status []int, body []string, wantRequest []*http.Request, test func(SplunkClient) error) { + mockSplunkClient := &spltest.MockHTTPClient{} + for i := 0; i < len(wantRequest); i++ { + mockSplunkClient.AddHandler(wantRequest[i], status[i], body[i], nil) + } + c := NewSplunkClient("https://localhost:8089", "admin", "p@ssw0rd") + c.Client = mockSplunkClient + err := test(*c) + if err != nil { + t.Errorf("%s err = %v", testMethod, err) + } + mockSplunkClient.CheckRequests(t, testMethod) +} + func TestGetSearchHeadCaptainInfo(t *testing.T) { wantRequest, _ := http.NewRequest("GET", "https://localhost:8089/services/shcluster/captain/info?count=0&output_mode=json", nil) wantCaptainLabel := "splunk-s2-search-head-0" @@ -345,6 +360,174 @@ func TestDecommissionIndexerClusterPeer(t *testing.T) { splunkClientTester(t, "TestDecommissionIndexerClusterPeer", 200, "", wantRequest, test) } +func TestAutomateMCApplyChanges(t *testing.T) { + request1, _ := http.NewRequest("GET", "https://localhost:8089/services/server/info/server-info?count=0&output_mode=json", nil) + request2, _ := http.NewRequest("GET", "https://localhost:8089/services/search/distributed/peers?count=0&output_mode=json", nil) + request3, _ := http.NewRequest("POST", "https://localhost:8089/services/search/distributed/groups/dmc_group_indexer/edit", nil) + request4, _ := http.NewRequest("POST", "https://localhost:8089/services/search/distributed/groups/dmc_group_license_master/edit", nil) + request5, _ := http.NewRequest("POST", "https://localhost:8089/services/search/distributed/groups/dmc_indexerclustergroup_idxc_label/edit", nil) + request6, _ := http.NewRequest("GET", "https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full?count=0&output_mode=json", nil) + request7, _ := http.NewRequest("POST", "https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/saved/searches/DMC%20Asset%20-%20Build%20Full/dispatch", nil) + request8, _ := http.NewRequest("GET", "https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed?count=0&output_mode=json", nil) + request9, _ := http.NewRequest("POST", "https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/configs/conf-splunk_monitoring_console_assets/settings", nil) + request10, _ := http.NewRequest("POST", "https://localhost:8089/servicesNS/nobody/system/apps/local/splunk_monitoring_console", nil) + var wantRequests []*http.Request + wantRequests = []*http.Request(append(wantRequests, request1, request2, request3, request4, request5, request6, request7, request8, request9, request10)) + body := []string{ + `{"links":{},"origin":"https://localhost:8089/services/server/info","updated":"2020-09-24T06:38:53+00:00","generator":{"build":"a1a6394cc5ae","version":"8.0.5"},"entry":[{"name":"server-info","id":"https://localhost:8089/services/server/info/server-info","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/server/info/server-info","list":"/services/server/info/server-info"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["*"],"write":[]},"removable":false,"sharing":"system"},"fields":{"required":[],"optional":[],"wildcard":[]},"content":{"activeLicenseGroup":"Trial","activeLicenseSubgroup":"Production","addOns":{"DFS":{"parameters":{"vCPU":"0"},"type":"add_on"},"hadoop":{"parameters":{"erp_type":"report","guid":"6F416E61-B40E-461C-A782-CBC186E98133","maxNodes":"200"},"type":"external_results_provider"}},"build":"a1a6394cc5ae","cluster_label":["idxc_label"],"cpu_arch":"x86_64","dfs_enabled":false,"eai:acl":null,"fips_mode":false,"guid":"0F93F33C-4BDA-4A74-AD9F-3FCE26C6AFF0","health_info":"green","health_version":1,"host":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","host_fqdn":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","host_resolved":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","isForwarding":true,"isFree":false,"isTrial":true,"kvStoreStatus":"ready","licenseKeys":["5C52DA5145AD67B8188604C49962D12F2C3B2CF1B82A6878E46F68CA2812807B"],"licenseSignature":"139bf73ec92c84121c79a9b8307a6724","licenseState":"OK","license_labels":["Splunk Enterprise Splunk Analytics for Hadoop Download Trial"],"master_guid":"0F93F33C-4BDA-4A74-AD9F-3FCE26C6AFF0","master_uri":"self","max_users":4294967295,"mode":"normal","numberOfCores":1,"numberOfVirtualCores":2,"os_build":"#1 SMP Thu Sep 3 19:04:44 UTC 2020","os_name":"Linux","os_name_extended":"Linux","os_version":"4.14.193-149.317.amzn2.x86_64","physicalMemoryMB":7764,"product_type":"enterprise","rtsearch_enabled":true,"serverName":"splunk-default-monitoring-console-86bc9b7c8c-d96x2","server_roles":["license_master","cluster_search_head","search_head"],"startup_time":1600928786,"staticAssetId":"CFE3D41EE2CCD1465E8C8453F83E4ECFFF540780B4490E84458DD4A3694CE4D1","version":"8.0.5"}}],"paging":{"total":1,"perPage":30,"offset":0},"messages":[]}`, + `{"links":{"create":"/services/search/distributed/peers/_new","_reload":"/services/search/distributed/peers/_reload"},"origin":"https://localhost:8089/services/search/distributed/peers","updated":"2020-09-22T21:43:33+00:00","generator":{"build":"a1a6394cc5ae","version":"8.0.5"},"entry":[{"name":"splunk-example-cluster-master-service:8089","id":"https://localhost:8089/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089","list":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089","_reload":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089/_reload","edit":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089","remove":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089","disable":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089/disable","quarantine":"/services/search/distributed/peers/splunk-example-cluster-master-service%3A8089/quarantine"},"author":"system","acl":{"app":"","can_list":true,"can_write":true,"modifiable":false,"owner":"system","perms":{"read":["admin","splunk-system-role"],"write":["admin","splunk-system-role"]},"removable":false,"sharing":"system"},"content":{"build":"a1a6394cc5ae","bundle_isIndexing":["15065687072217053252 - false","16923722576385601869 - false","8527338655074501134 - false","5466829646820181037 - false","12887664000706842849 - false"],"bundle_versions":["15065687072217053252","16923722576385601869","8527338655074501134","5466829646820181037","12887664000706842849"],"cluster_label":["idxc_label"],"cpu_arch":"x86_64","disabled":false,"eai:acl":null,"enableRFSMonitoring":false,"guid":"19F5CA13-A561-4D1B-9341-A7BFC3E92F06","host":"splunk-example-cluster-master-0","host_fqdn":"splunk-example-cluster-master-0","isForwarding":true,"is_https":true,"licenseSignature":"c2bf3f573bbaf36ae13f326762945905","numberOfCores":"1","numberOfVirtualCores":"2","os_build":"#1 SMP Thu Sep 3 19:04:44 UTC 2020","os_name":"Linux","os_version":"4.14.193-149.317.amzn2.x86_64","peerName":"splunk-example-cluster-master-0","peerType":"configured","physicalMemoryMB":"7764","remote_session":"2bOC3nPGg_s2UREElAhLcKFXxuMNgwasCQPJG^tlXJ5LMzX1aCC8zwgWN2dCtUD2TyQ1szY0sdQhOwJyK_Igj6JUfEmcVGrsd3BtBvFOb0R_6F9kYRzHKz3","replicationStatus":"Successful","rtsearch_enabled":true,"search_groups":["dmc_group_cluster_master","dmc_group_license_master","dmc_indexerclustergroup_idxc_label"],"searchable_indexes":["_audit","_internal","_introspection","_metrics","_metrics_rollup","_telemetry","_thefishbucket","history","main","summary"],"server_roles":["license_master","cluster_master","search_head"],"shcluster_label":"","startup_time":1600810341,"status":"Up","status_details":[],"version":"8.0.5"}}],"paging":{"total":4,"perPage":30,"offset":0},"messages":[]}`, + "", + "", + "", + `{"links":{"create":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_new","_reload":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_reload","_acl":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/_acl"},"origin":"https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/data/ui/nav","updated":"2020-09-23T23:50:25+00:00","generator":{"build":"a1a6394cc5ae","version":"8.0.5"},"entry":[{"name":"default.distributed","id":"https://localhost:8089/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","updated":"1970-01-01T00:00:00+00:00","links":{"alternate":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","list":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed","_reload":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed/_reload","edit":"/servicesNS/nobody/splunk_monitoring_console/data/ui/nav/default.distributed"},"author":"nobody","acl":{"app":"splunk_monitoring_console","can_change_perms":true,"can_list":true,"can_share_app":true,"can_share_global":true,"can_share_user":false,"can_write":true,"modifiable":true,"owner":"nobody","perms":{"read":["admin"],"write":["admin"]},"removable":false,"sharing":"app"},"fields":{"required":["eai:data"],"optional":[],"wildcard":[]},"content":{"disabled":false,"eai:acl":null,"eai:appName":"splunk_monitoring_console","eai:data":"