[Backport 2.x] security implementation

.github/workflows/security-test-workflow.yml

@@ -0,0 +1,88 @@
+name: Security Test Workflow
+# This workflow is triggered on pull requests and pushes to main or an OpenSearch release branch
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "*"
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        java: [ 11, 17 ]
+    # Job name
+    name: Build and test SecurityAnalytics
+    # This job runs on Linux
+    runs-on: ubuntu-latest
+    steps:
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      # This step uses the checkout Github action: https://github.com/actions/checkout
+      - name: Checkout Branch
+        uses: actions/checkout@v2
+      # This step uses the setup-java Github action: https://github.com/actions/setup-java
+      - name: Set Up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Build SecurityAnalytics
+        # Only assembling since the full build is governed by other workflows
+        run: ./gradlew assemble
+
+      - name: Pull and Run Docker
+        run: |
+          plugin=`basename $(ls build/distributions/*.zip)`
+          list_of_files=`ls`
+          list_of_all_files=`ls build/distributions/`
+          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
+          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
+          qualifier=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-1`
+          candidate_version=`echo $plugin|awk -F- '{print $5}'| cut -d. -f 1-1`
+          docker_version=$version
+
+          [[ -z $candidate_version ]] && candidate_version=$qualifier && qualifier=""
+
+          echo plugin version plugin_version qualifier candidate_version docker_version
+          echo "($plugin) ($version) ($plugin_version) ($qualifier) ($candidate_version) ($docker_version)"
+          echo $ls $list_of_all_files
+
+          if docker pull opensearchstaging/opensearch:$docker_version
+          then
+            echo "FROM opensearchstaging/opensearch:$docker_version" >> Dockerfile
+            echo "RUN if [ -d /usr/share/opensearch/plugins/opensearch-security-analytics ]; then /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security-analytics; fi" >> Dockerfile
+            echo "ADD build/distributions/$plugin /tmp/" >> Dockerfile
+            echo "RUN /usr/share/opensearch/bin/opensearch-plugin install --batch file:/tmp/$plugin" >> Dockerfile
+
+            docker build -t opensearch-security-analytics:test .
+            echo "imagePresent=true" >> $GITHUB_ENV
+          else
+            echo "imagePresent=false" >> $GITHUB_ENV
+          fi
+
+      - name: Run Docker Image
+        if: env.imagePresent == 'true'
+        run: |
+          cd ..
+          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" opensearch-security-analytics:test
+          sleep 120
+
+      - name: Run SecurityAnalytics Test for security enabled test cases
+        if: env.imagePresent == 'true'
+        run: |
+          cluster_running=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure`
+          echo $cluster_running
+          security=`curl -XGET https://localhost:9200/_cat/plugins -u admin:admin --insecure |grep opensearch-security|wc -l`
+          echo $security
+          if [ $security -gt 0 ]
+          then
+            echo "Security plugin is available"
+            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Dsecurity=true -Dhttps=true -Duser=admin -Dpassword=admin          
+          else
+            echo "Security plugin is NOT available skipping this run as tests without security have already been run"
+          fi

build.gradle

@@ -165,8 +165,24 @@ integTest {
     systemProperty 'java.io.tmpdir', es_tmp_dir.absolutePath
 
     systemProperty "https", System.getProperty("https")
+    systemProperty "security", System.getProperty("security")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
+
+    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.securityanalytics.resthandler.Secure*RestApiIT"
+            excludeTestsMatching "org.opensearch.securityanalytics.findings.Secure*RestApiIT"
+            excludeTestsMatching "org.opensearch.securityanalytics.alerts.Secure*RestApiIT"
+        }
+    }
+
+    if (System.getProperty("https") != null || System.getProperty("https") == "true") {
+        filter {
+            excludeTestsMatching "org.opensearch.securityanalytics.*TransportIT"
+        }
+    }
+
     // Tell the test JVM if the cluster JVM is running under a debugger so that tests can use longer timeouts for
     // requests. The 'doFirst' delays reading the debug setting on the cluster till execution time.
     doFirst {
@@ -299,7 +315,15 @@ task integTestRemote(type: RestIntegTestTask) {
 
     if (System.getProperty("tests.rest.cluster") != null) {
         filter {
-            includeTestsMatching "org.opensearch.securityanalytics.*RestIT"
+            includeTestsMatching "org.opensearch.securityanalytics.*RestApiIT"
+        }
+    }
+
+    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
+        filter {
+            excludeTestsMatching "org.opensearch.securityanalytics.resthandler.Secure*RestApiIT"
+            excludeTestsMatching "org.opensearch.securityanalytics.findings.Secure*RestApiIT"
+            excludeTestsMatching "org.opensearch.securityanalytics.alerts.Secure*RestApiIT"
         }
     }
 }

src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java

@@ -171,14 +171,14 @@ public List<NamedXContentRegistry.Entry> getNamedXContent() {
     public List<Setting<?>> getSettings() {
         return List.of(
                 SecurityAnalyticsSettings.INDEX_TIMEOUT,
+                SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES,
                 SecurityAnalyticsSettings.ALERT_HISTORY_ENABLED,
                 SecurityAnalyticsSettings.ALERT_HISTORY_ROLLOVER_PERIOD,
                 SecurityAnalyticsSettings.ALERT_HISTORY_INDEX_MAX_AGE,
                 SecurityAnalyticsSettings.ALERT_HISTORY_MAX_DOCS,
                 SecurityAnalyticsSettings.ALERT_HISTORY_RETENTION_PERIOD,
                 SecurityAnalyticsSettings.REQUEST_TIMEOUT,
                 SecurityAnalyticsSettings.MAX_ACTION_THROTTLE_VALUE,
-                SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES,
                 SecurityAnalyticsSettings.FINDING_HISTORY_ENABLED,
                 SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS,
                 SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,

src/main/java/org/opensearch/securityanalytics/action/AckAlertsAction.java

@@ -7,7 +7,7 @@
 import org.opensearch.action.ActionType;
 
 public class AckAlertsAction extends ActionType<AckAlertsResponse> {
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/alerts/ack";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/alerts/ack";
     public static final AckAlertsAction INSTANCE = new AckAlertsAction();
 
     public AckAlertsAction() {

src/main/java/org/opensearch/securityanalytics/action/CreateIndexMappingsAction.java

@@ -9,7 +9,7 @@
 
 public class CreateIndexMappingsAction extends ActionType<AcknowledgedResponse>{
 
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/mapping/create";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/mapping/create";
     public static final CreateIndexMappingsAction INSTANCE = new CreateIndexMappingsAction();
 
 

src/main/java/org/opensearch/securityanalytics/action/DeleteDetectorAction.java

@@ -9,7 +9,7 @@
 public class DeleteDetectorAction extends ActionType<DeleteDetectorResponse> {
 
     public static final DeleteDetectorAction INSTANCE = new DeleteDetectorAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/detector/delete";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/detector/delete";
 
     public DeleteDetectorAction() {
         super(NAME, DeleteDetectorResponse::new);

src/main/java/org/opensearch/securityanalytics/action/DeleteRuleAction.java

@@ -9,7 +9,7 @@
 public class DeleteRuleAction extends ActionType<DeleteRuleResponse> {
 
     public static final DeleteRuleAction INSTANCE = new DeleteRuleAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/rule/delete";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/rule/delete";
 
     public DeleteRuleAction() {
         super(NAME, DeleteRuleResponse::new);

src/main/java/org/opensearch/securityanalytics/action/GetAlertsAction.java

@@ -9,7 +9,7 @@
 public class GetAlertsAction extends ActionType<GetAlertsResponse> {
 
     public static final GetAlertsAction INSTANCE = new GetAlertsAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/alerts/get";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/alerts/get";
 
     public GetAlertsAction() {
         super(NAME, GetAlertsResponse::new);

src/main/java/org/opensearch/securityanalytics/action/GetDetectorAction.java

@@ -9,7 +9,7 @@
 public class GetDetectorAction extends ActionType<GetDetectorResponse> {
 
     public static final GetDetectorAction INSTANCE = new GetDetectorAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/detector/get";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/detector/get";
 
     public GetDetectorAction() {
         super(NAME, GetDetectorResponse::new);

src/main/java/org/opensearch/securityanalytics/action/GetFindingsAction.java

@@ -9,7 +9,7 @@
 public class GetFindingsAction extends ActionType<GetFindingsResponse> {
 
     public static final GetFindingsAction INSTANCE = new GetFindingsAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/findings/get";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/findings/get";
 
     public GetFindingsAction() {
         super(NAME, GetFindingsResponse::new);

src/main/java/org/opensearch/securityanalytics/action/GetIndexMappingsAction.java

@@ -8,7 +8,7 @@
 
 public class GetIndexMappingsAction extends ActionType<GetIndexMappingsResponse>{
 
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/mapping/get";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/mapping/get";
     public static final GetIndexMappingsAction INSTANCE = new GetIndexMappingsAction();
 
 

src/main/java/org/opensearch/securityanalytics/action/GetMappingsViewAction.java

@@ -8,7 +8,7 @@
 
 public class GetMappingsViewAction extends ActionType<GetMappingsViewResponse>{
 
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/mapping/view/get";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/mapping/view/get";
     public static final GetMappingsViewAction INSTANCE = new GetMappingsViewAction();
 
 

src/main/java/org/opensearch/securityanalytics/action/IndexDetectorAction.java

@@ -9,7 +9,7 @@
 public class IndexDetectorAction extends ActionType<IndexDetectorResponse> {
 
     public static final IndexDetectorAction INSTANCE = new IndexDetectorAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/detector/write";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/detector/write";
 
     public IndexDetectorAction() {
         super(NAME, IndexDetectorResponse::new);

src/main/java/org/opensearch/securityanalytics/action/IndexRuleAction.java

@@ -9,7 +9,7 @@
 public class IndexRuleAction extends ActionType<IndexRuleResponse> {
 
     public static final IndexRuleAction INSTANCE = new IndexRuleAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/rule/write";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/rule/write";
 
     public IndexRuleAction() {
         super(NAME, IndexRuleResponse::new);

src/main/java/org/opensearch/securityanalytics/action/SearchDetectorAction.java

@@ -10,7 +10,7 @@
 public class SearchDetectorAction extends ActionType<SearchResponse> {
 
     public static final SearchDetectorAction INSTANCE = new SearchDetectorAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/detector/search";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/detector/search";
 
     public SearchDetectorAction() {
         super(NAME, SearchResponse::new);

src/main/java/org/opensearch/securityanalytics/action/SearchRuleAction.java

@@ -10,7 +10,7 @@
 public class SearchRuleAction extends ActionType<SearchResponse> {
 
     public static final SearchRuleAction INSTANCE = new SearchRuleAction();
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/rule/search";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/rule/search";
 
     public SearchRuleAction() {
         super(NAME, SearchResponse::new);

src/main/java/org/opensearch/securityanalytics/action/UpdateIndexMappingsAction.java

@@ -9,7 +9,7 @@
 
 public class UpdateIndexMappingsAction extends ActionType<AcknowledgedResponse>{
 
-    public static final String NAME = "cluster:admin/opendistro/securityanalytics/mapping/update";
+    public static final String NAME = "cluster:admin/opensearch/securityanalytics/mapping/update";
     public static final UpdateIndexMappingsAction INSTANCE = new UpdateIndexMappingsAction();
 
 

src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java

@@ -6,6 +6,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchStatusException;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.action.support.WriteRequest;
@@ -17,6 +18,7 @@
 import org.opensearch.commons.alerting.action.GetAlertsRequest;
 import org.opensearch.commons.alerting.model.Alert;
 import org.opensearch.commons.alerting.model.Table;
+import org.opensearch.rest.RestStatus;
 import org.opensearch.securityanalytics.action.AckAlertsResponse;
 import org.opensearch.securityanalytics.action.AlertDto;
 import org.opensearch.securityanalytics.action.GetAlertsResponse;
@@ -102,7 +104,7 @@ public void onFailure(Exception e) {
 
             @Override
             public void onFailure(Exception e) {
-                listener.onFailure(SecurityAnalyticsException.wrap(e));
+                listener.onFailure(e);
             }
         });
     }
@@ -172,7 +174,7 @@ public void getAlerts(
             ActionListener<GetAlertsResponse> listener
     ) {
         if (detectors.size() == 0) {
-            throw SecurityAnalyticsException.wrap(new IllegalArgumentException("detector list is empty!"));
+            throw new OpenSearchStatusException("detector list is empty!", RestStatus.NOT_FOUND);
         }
 
         List<String> allMonitorIds = new ArrayList<>();

src/main/java/org/opensearch/securityanalytics/findings/FindingsService.java

@@ -12,6 +12,7 @@
 import java.util.stream.Collectors;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchStatusException;
 import org.opensearch.action.ActionListener;
 import org.opensearch.action.support.GroupedActionListener;
 import org.opensearch.client.Client;
@@ -51,7 +52,7 @@ public FindingsService(Client client) {
      * @param table group of search related parameters
      * @param listener ActionListener to get notified on response or error
      */
-    public void getFindingsByDetectorId(String detectorId, Table table, ActionListener<GetFindingsResponse> listener) {
+    public void getFindingsByDetectorId(String detectorId, Table table, ActionListener<GetFindingsResponse> listener ) {
         this.client.execute(GetDetectorAction.INSTANCE, new GetDetectorRequest(detectorId, -3L), new ActionListener<>() {
 
             @Override
@@ -102,7 +103,7 @@ public void onFailure(Exception e) {
 
             @Override
             public void onFailure(Exception e) {
-                listener.onFailure(SecurityAnalyticsException.wrap(e));
+                listener.onFailure(e);
             }
         });
     }
@@ -167,7 +168,7 @@ public void getFindings(
             ActionListener<GetFindingsResponse> listener
     ) {
         if (detectors.size() == 0) {
-            throw SecurityAnalyticsException.wrap(new IllegalArgumentException("detector list is empty!"));
+            throw new OpenSearchStatusException("detector list is empty!", RestStatus.NOT_FOUND);
         }
 
         List<String> allMonitorIds = new ArrayList<>();

src/main/java/org/opensearch/securityanalytics/model/Detector.java

@@ -212,7 +212,8 @@ public enum DetectorType {
         APACHE_ACCESS("apache_access"),
         CLOUDTRAIL("cloudtrail"),
         DNS("dns"),
-        S3("s3");
+        S3("s3"),
+        TEST_WINDOWS("test_windows");
 
         private String type;
 
@@ -516,6 +517,10 @@ public List<String> getMonitorIds() {
         return monitorIds;
     }
 
+    public void setUser(User user) {
+        this.user = user;
+    }
+
     public void setId(String id) {
         this.id = id;
     }

src/main/java/org/opensearch/securityanalytics/settings/SecurityAnalyticsSettings.java

@@ -90,7 +90,7 @@ public class SecurityAnalyticsSettings {
             Setting.Property.NodeScope, Setting.Property.Dynamic
     );
 
-    public static final Setting FILTER_BY_BACKEND_ROLES = Setting.boolSetting(
+    public static final Setting<Boolean> FILTER_BY_BACKEND_ROLES = Setting.boolSetting(
             "plugins.security_analytics.filter_by_backend_roles",
             false,
             Setting.Property.NodeScope, Setting.Property.Dynamic