Change AD indices to be hidden indices instead of system indices.

.github/workflows/CI.yml

@@ -4,10 +4,12 @@ on:
     branches:
       - main
       - opendistro-*
+      - 7.10.2-no-workbench
   pull_request:
     branches:
       - main
       - opendistro-*
+      - 7.10.2-no-workbench
 
 jobs:
   Build-ad:

src/main/java/com/amazon/opendistroforelasticsearch/ad/AnomalyDetectorPlugin.java

@@ -50,12 +50,10 @@
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
-import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.monitor.jvm.JvmService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
-import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
@@ -170,7 +168,7 @@
 /**
  * Entry point of AD plugin.
  */
-public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension, SystemIndexPlugin {
+public class AnomalyDetectorPlugin extends Plugin implements ActionPlugin, ScriptPlugin, JobSchedulerExtension {
 
     private static final Logger LOG = LogManager.getLogger(AnomalyDetectorPlugin.class);
 
@@ -635,19 +633,4 @@ public ScheduledJobParser getJobParser() {
             return AnomalyDetectorJob.parse(parser);
         };
     }
-
-    @Override
-    public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
-        return Collections
-            .unmodifiableList(
-                Arrays
-                    .asList(
-                        new SystemIndexDescriptor(AnomalyDetectionIndices.ALL_AD_RESULTS_INDEX_PATTERN, "anomaly result"),
-                        new SystemIndexDescriptor(AnomalyDetector.ANOMALY_DETECTORS_INDEX, "detector definition"),
-                        new SystemIndexDescriptor(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX, "detector job"),
-                        new SystemIndexDescriptor(CommonName.CHECKPOINT_INDEX_NAME, "model checkpoint"),
-                        new SystemIndexDescriptor(DetectorInternalState.DETECTOR_STATE_INDEX, "detector information like total rcf updates")
-                    )
-            );
-    }
 }

src/main/java/com/amazon/opendistroforelasticsearch/ad/indices/AnomalyDetectionIndices.java

@@ -114,6 +114,8 @@ public class AnomalyDetectionIndices implements LocalNodeMasterListener {
     private boolean allUpdated;
     // we only want one update at a time
     private final AtomicBoolean updateRunning;
+    // AD index settings
+    private final Settings setting;
 
     class IndexState {
         // keep track of whether the mapping version is up-to-date
@@ -170,6 +172,8 @@ public AnomalyDetectionIndices(
             .addSettingsUpdateConsumer(AD_RESULT_HISTORY_RETENTION_PERIOD, it -> { historyRetentionPeriod = it; });
 
         this.clusterService.getClusterSettings().addSettingsUpdateConsumer(MAX_PRIMARY_SHARDS, it -> maxPrimaryShards = it);
+
+        this.setting = Settings.builder().put("index.hidden", true).build();
     }
 
     /**
@@ -325,7 +329,8 @@ public void initAnomalyDetectorIndexIfAbsent(ActionListener<CreateIndexResponse>
      */
     public void initAnomalyDetectorIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         CreateIndexRequest request = new CreateIndexRequest(AnomalyDetector.ANOMALY_DETECTORS_INDEX)
-            .mapping(AnomalyDetector.TYPE, getAnomalyDetectorMappings(), XContentType.JSON);
+            .mapping(AnomalyDetector.TYPE, getAnomalyDetectorMappings(), XContentType.JSON)
+            .settings(setting);
         adminClient.indices().create(request, markMappingUpToDate(ADIndex.CONFIG, actionListener));
     }
 
@@ -367,6 +372,7 @@ public void initAnomalyResultIndexDirectly(ActionListener<CreateIndexResponse> a
         String mapping = getAnomalyResultMappings();
         CreateIndexRequest request = new CreateIndexRequest(AD_RESULT_HISTORY_INDEX_PATTERN)
             .mapping(CommonName.MAPPING_TYPE, mapping, XContentType.JSON)
+            .settings(setting)
             .alias(new Alias(CommonName.ANOMALY_RESULT_INDEX_ALIAS));
         choosePrimaryShards(request);
         adminClient.indices().create(request, markMappingUpToDate(ADIndex.RESULT, actionListener));
@@ -381,7 +387,8 @@ public void initAnomalyResultIndexDirectly(ActionListener<CreateIndexResponse> a
     public void initAnomalyDetectorJobIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         // TODO: specify replica setting
         CreateIndexRequest request = new CreateIndexRequest(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX)
-            .mapping(AnomalyDetector.TYPE, getAnomalyDetectorJobMappings(), XContentType.JSON);
+            .mapping(AnomalyDetector.TYPE, getAnomalyDetectorJobMappings(), XContentType.JSON)
+            .settings(setting);
         choosePrimaryShards(request);
         adminClient.indices().create(request, markMappingUpToDate(ADIndex.JOB, actionListener));
     }
@@ -394,7 +401,8 @@ public void initAnomalyDetectorJobIndex(ActionListener<CreateIndexResponse> acti
      */
     public void initDetectorStateIndex(ActionListener<CreateIndexResponse> actionListener) throws IOException {
         CreateIndexRequest request = new CreateIndexRequest(DetectorInternalState.DETECTOR_STATE_INDEX)
-            .mapping(AnomalyDetector.TYPE, getDetectorStateMappings(), XContentType.JSON);
+            .mapping(AnomalyDetector.TYPE, getDetectorStateMappings(), XContentType.JSON)
+            .settings(setting);
         adminClient.indices().create(request, markMappingUpToDate(ADIndex.STATE, actionListener));
     }
 
@@ -412,7 +420,8 @@ public void initCheckpointIndex(ActionListener<CreateIndexResponse> actionListen
             throw new EndRunException("", "Cannot find checkpoint mapping file", true);
         }
         CreateIndexRequest request = new CreateIndexRequest(CommonName.CHECKPOINT_INDEX_NAME)
-            .mapping(CommonName.MAPPING_TYPE, mapping, XContentType.JSON);
+            .mapping(CommonName.MAPPING_TYPE, mapping, XContentType.JSON)
+            .settings(setting);
         choosePrimaryShards(request);
         adminClient.indices().create(request, markMappingUpToDate(ADIndex.CHECKPOINT, actionListener));
     }
@@ -470,7 +479,9 @@ void rolloverAndDeleteHistoryIndex() {
         request
             .getCreateIndexRequest()
             .index(AD_RESULT_HISTORY_INDEX_PATTERN)
-            .mapping(CommonName.MAPPING_TYPE, adResultMapping, XContentType.JSON);
+            .mapping(CommonName.MAPPING_TYPE, adResultMapping, XContentType.JSON)
+            .settings(setting);
+
         request.addMaxIndexDocsCondition(historyMaxDocs);
         adminClient.indices().rolloverIndex(request, ActionListener.wrap(response -> {
             if (!response.isRolledOver()) {

src/test/java/com/amazon/opendistroforelasticsearch/ad/TestHelpers.java

@@ -159,7 +159,7 @@ public static Response makeRequest(
         HttpEntity entity,
         List<Header> headers
     ) throws IOException {
-        return makeRequest(client, method, endpoint, params, entity, headers, true);
+        return makeRequest(client, method, endpoint, params, entity, headers, false);
     }
 
     public static Response makeRequest(

src/test/java/com/amazon/opendistroforelasticsearch/ad/e2e/DetectionResultEvalutationIT.java

@@ -15,6 +15,8 @@
 
 package com.amazon.opendistroforelasticsearch.ad.e2e;
 
+import static com.amazon.opendistroforelasticsearch.ad.TestHelpers.toHttpEntity;
+
 import java.io.File;
 import java.io.FileReader;
 import java.time.Instant;
@@ -29,20 +31,27 @@
 import java.util.Map.Entry;
 import java.util.Set;
 
+import org.apache.http.HttpHeaders;
+import org.apache.http.message.BasicHeader;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.WarningsHandler;
 
 import com.amazon.opendistroforelasticsearch.ad.ODFERestTestCase;
+import com.amazon.opendistroforelasticsearch.ad.TestHelpers;
+import com.google.common.collect.ImmutableList;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
 
 public class DetectionResultEvalutationIT extends ODFERestTestCase {
 
     public void testDataset() throws Exception {
-        verifyAnomaly("synthetic", 1, 1500, 8, .9, .9, 10);
+        // TODO: this test case will run for a much longer time and timeout with security enabled
+        if (!isHttps()) {
+            verifyAnomaly("synthetic", 1, 1500, 8, .9, .9, 10);
+        }
     }
 
     private void verifyAnomaly(
@@ -54,7 +63,6 @@ private void verifyAnomaly(
         double minRecall,
         double maxError
     ) throws Exception {
-
         RestClient client = client();
 
         String dataFileName = String.format("data/%s.data", datasetName);
@@ -63,11 +71,10 @@ private void verifyAnomaly(
         List<JsonObject> data = getData(dataFileName);
         List<Entry<Instant, Instant>> anomalies = getAnomalyWindows(labelFileName);
 
-        indexTrainData(datasetName, data, trainTestSplit, client);
+        bulkIndexTrainData(datasetName, data, trainTestSplit, client);
         String detectorId = createDetector(datasetName, intervalMinutes, client);
         startDetector(detectorId, data, trainTestSplit, shingleSize, intervalMinutes, client);
-
-        indexTestData(data, datasetName, trainTestSplit, client);
+        bulkIndexTestData(data, datasetName, trainTestSplit, client);
         double[] testResults = getTestResults(detectorId, data, trainTestSplit, intervalMinutes, anomalies, client);
         verifyTestResults(testResults, anomalies, minPrecision, minRecall, maxError);
     }
@@ -141,22 +148,6 @@ private double[] getTestResults(
         return new double[] { positives, truePositives, positiveAnomalies.size(), errors };
     }
 
-    private void indexTestData(List<JsonObject> data, String datasetName, int trainTestSplit, RestClient client) throws Exception {
-        data.stream().skip(trainTestSplit).forEach(r -> {
-            try {
-                Request req = new Request("POST", String.format("/%s/_doc/", datasetName));
-                RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
-                options.setWarningsHandler(WarningsHandler.PERMISSIVE);
-                req.setOptions(options.build());
-                req.setJsonEntity(r.toString());
-                client.performRequest(req);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
-        Thread.sleep(1_000);
-    }
-
     private void startDetector(
         String detectorId,
         List<JsonObject> data,
@@ -229,26 +220,56 @@ private List<Entry<Instant, Instant>> getAnomalyWindows(String labalFileName) th
         return anomalies;
     }
 
-    private void indexTrainData(String datasetName, List<JsonObject> data, int trainTestSplit, RestClient client) throws Exception {
+    private void bulkIndexTrainData(String datasetName, List<JsonObject> data, int trainTestSplit, RestClient client) throws Exception {
         Request request = new Request("PUT", datasetName);
         String requestBody = "{ \"mappings\": { \"properties\": { \"timestamp\": { \"type\": \"date\"},"
             + " \"Feature1\": { \"type\": \"double\" }, \"Feature2\": { \"type\": \"double\" } } } }";
         request.setJsonEntity(requestBody);
+        setWarningHandler(request, false);
         client.performRequest(request);
         Thread.sleep(1_000);
 
-        data.stream().limit(trainTestSplit).forEach(r -> {
-            try {
-                Request req = new Request("POST", String.format("/%s/_doc/", datasetName));
-                req.setJsonEntity(r.toString());
-                client.performRequest(req);
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
-        });
+        StringBuilder bulkRequestBuilder = new StringBuilder();
+        for (int i = 0; i < trainTestSplit; i++) {
+            bulkRequestBuilder.append("{ \"index\" : { \"_index\" : \"" + datasetName + "\", \"_id\" : \"" + i + "\" } }\n");
+            bulkRequestBuilder.append(data.get(i).toString()).append("\n");
+        }
+        TestHelpers
+            .makeRequest(
+                client,
+                "POST",
+                "_bulk?refresh=true",
+                null,
+                toHttpEntity(bulkRequestBuilder.toString()),
+                ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"))
+            );
         Thread.sleep(1_000);
     }
 
+    private void bulkIndexTestData(List<JsonObject> data, String datasetName, int trainTestSplit, RestClient client) throws Exception {
+        StringBuilder bulkRequestBuilder = new StringBuilder();
+        for (int i = trainTestSplit; i < data.size(); i++) {
+            bulkRequestBuilder.append("{ \"index\" : { \"_index\" : \"" + datasetName + "\", \"_id\" : \"" + i + "\" } }\n");
+            bulkRequestBuilder.append(data.get(i).toString()).append("\n");
+        }
+        TestHelpers
+            .makeRequest(
+                client,
+                "POST",
+                "_bulk?refresh=true",
+                null,
+                toHttpEntity(bulkRequestBuilder.toString()),
+                ImmutableList.of(new BasicHeader(HttpHeaders.USER_AGENT, "Kibana"))
+            );
+        Thread.sleep(1_000);
+    }
+
+    private void setWarningHandler(Request request, boolean strictDeprecationMode) {
+        RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
+        options.setWarningsHandler(strictDeprecationMode ? WarningsHandler.STRICT : WarningsHandler.PERMISSIVE);
+        request.setOptions(options.build());
+    }
+
     private List<JsonObject> getData(String datasetFileName) throws Exception {
         JsonArray jsonArray = new JsonParser()
             .parse(new FileReader(new File(getClass().getResource(datasetFileName).toURI())))