ES-Operator doesn't separate system indices and ordinary users indices.

Closes #161

Signed-off-by: Oliver <oliver.trosien@zalando.de>

docs/zalando.org_elasticsearchdatasets.yaml

@@ -45,6 +45,9 @@ spec:
           spec:
             description: ElasticsearchDataSetSpec is the spec part of the Elasticsearch dataset.
             properties:
+              excludeSystemIndices:
+                description: Exclude management of System Indices on this Data Set. Defaults to false
+                type: boolean
               replicas:
                 description: Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
                 format: int32

docs/zalando.org_elasticsearchdatasets_v1beta1.yaml

@@ -46,6 +46,9 @@ spec:
         spec:
           description: ElasticsearchDataSetSpec is the spec part of the Elasticsearch dataset.
           properties:
+            excludeSystemIndices:
+              description: Exclude management of System Indices on this Data Set. Defaults to false
+              type: boolean
             replicas:
               description: Number of desired pods. This is a pointer to distinguish between explicit zero and not specified. Defaults to 1.
               format: int32

operator/elasticsearch.go

@@ -236,6 +236,7 @@ func (o *ElasticsearchOperator) runAutoscaler(ctx context.Context) {
 
 					client := &ESClient{
 						Endpoint: endpoint,
+						excludeSystemIndices: es.ElasticsearchDataSet.Spec.ExcludeSystemIndices,
 					}
 
 					err := o.scaleEDS(ctx, es.ElasticsearchDataSet, es, client)

operator/es_client.go

@@ -26,8 +26,9 @@ var (
 
 // ESClient is a pod drainer which can drain data from Elasticsearch pods.
 type ESClient struct {
-	Endpoint *url.URL
-	mux      sync.Mutex
+	Endpoint             *url.URL
+	mux                  sync.Mutex
+	excludeSystemIndices bool
 }
 
 // ESIndex represent an index to be used in public APIs
@@ -388,9 +389,13 @@ func (c *ESClient) GetIndices() ([]ESIndex, error) {
 		return nil, err
 	}
 
-	// HACK: after-the-fact conversion of strings to integers from ES response.
 	returnStruct := make([]ESIndex, 0, len(esIndices))
 	for _, index := range esIndices {
+		// ignore system indices
+		if c.excludeSystemIndices && index.Index[0:1] == "." {
+			continue
+		}
+		// HACK: after-the-fact conversion of strings to integers from ES response.
 		primaries, err := strconv.Atoi(index.Primaries)
 		if err != nil {
 			return nil, err

operator/es_client_test.go

@@ -129,10 +129,10 @@ func TestGetIndices(t *testing.T) {
 
 	assert.NoError(t, err)
 
-	require.EqualValues(t, 3, len(indices))
-	require.EqualValues(t, "a", indices[0].Index)
-	require.EqualValues(t, 2, indices[0].Primaries)
-	require.EqualValues(t, 1, indices[0].Replicas)
+	require.EqualValues(t, 3, len(indices), indices)
+	require.EqualValues(t, "a", indices[0].Index, indices)
+	require.EqualValues(t, 2, indices[0].Primaries, indices)
+	require.EqualValues(t, 1, indices[0].Replicas, indices)
 
 }
 
@@ -239,3 +239,23 @@ func TestEnsureGreenClusterState(t *testing.T) {
 
 	assert.Error(t, err)
 }
+
+func TestExcludeSystemIndices(t *testing.T) {
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+
+	httpmock.RegisterResponder("GET", "http://elasticsearch:9200/_cat/indices",
+		httpmock.NewStringResponder(200, `[{"index":".system","pri":"1","rep":"1"},{"index":"a","pri":"1","rep":"1"}]`))
+
+	url, _ := url.Parse("http://elasticsearch:9200")
+	systemUnderTest := &ESClient{
+		Endpoint: url,
+		excludeSystemIndices: true,
+	}
+	indices, err := systemUnderTest.GetIndices()
+
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(indices), indices)
+	assert.Equal(t, "a", indices[0].Index, indices)
+
+}

pkg/apis/zalando.org/v1/types.go

@@ -33,6 +33,10 @@ type ElasticsearchDataSetSpec struct {
 	// +optional
 	Replicas *int32 `json:"replicas,omitempty" protobuf:"varint,1,opt,name=replicas"`
 
+	// Exclude management of System Indices on this Data Set. Defaults to false
+	// +optional
+	ExcludeSystemIndices bool `json:"excludeSystemIndices"`
+
 	// // serviceName is the name of the service that governs this StatefulSet.
 	// // This service must exist before the StatefulSet, and is responsible for
 	// // the network identity of the set. Pods get DNS/hostnames that follow the