Skip to content

Commit

Permalink
Backport changes and Integ test fix (#935)
Browse files Browse the repository at this point in the history
* Fix duplicate ecs mappings which returns incorrect log index field in mapping view API (#786) (#788)

* field mapping changes

Signed-off-by: Joanne Wang <jowg@amazon.com>

* add integ test

Signed-off-by: Joanne Wang <jowg@amazon.com>

* turn unmappedfieldaliases as set and add integ test

Signed-off-by: Joanne Wang <jowg@amazon.com>

* add comments

Signed-off-by: Joanne Wang <jowg@amazon.com>

* fix integ tests

Signed-off-by: Joanne Wang <jowg@amazon.com>

* moved logic to method for better readability

Signed-off-by: Joanne Wang <jowg@amazon.com>

---------

Signed-off-by: Joanne Wang <jowg@amazon.com>

* support object fields in aggregation based sigma rules (#789)

Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>

* Pass rule field names in doc level queries during monitor/creation. Remove blocking actionGet() calls  (#873)

* pass query field names in doc level queries during monitor creation/updation

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* remove actionGet() and change get index mapping call to event driven flow

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* fix chained findings monitor

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* add finding mappings

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* remove test messages from logs

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* revert build.gradle change

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* version bump

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* AggregationBackendTests fix

Signed-off-by: Riya Saxena <riysaxen@amazon.com>

---------

Signed-off-by: Joanne Wang <jowg@amazon.com>
Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>
Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
Signed-off-by: Riya Saxena <riysaxen@amazon.com>
Co-authored-by: Joanne Wang <jowg@amazon.com>
Co-authored-by: Subhobrata Dey <sbcd90@gmail.com>
Co-authored-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
4 people committed Mar 16, 2024
1 parent 7fbe7c0 commit e44e5d4
Show file tree
Hide file tree
Showing 12 changed files with 1,233 additions and 344 deletions.
2 changes: 1 addition & 1 deletion build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.9.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.9.1-SNAPSHOT")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
version_tokens = opensearch_version.tokenize('-')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) {
"\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+
"\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+
"\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+
"\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+
"\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+
"\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+
"\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+
"\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+
"\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+
"\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+
"\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+
"\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+
"\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+
"&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+
"\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+
"\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+
"\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+
"\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+
"\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+
">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+
"=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+
"\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+
"\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+
"\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+
"u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+
"\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+
"\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+
"\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+
"\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+
"\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+
"C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+
"F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+
"\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+
"-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+
"\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+
"PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+
"\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+
"\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+
"\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+
"\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+
"\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+
"\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+
"\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+
"\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+
"\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+
"\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+
"\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
"\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+
"z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+
"\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+
"\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+
"\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+
"\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+
"\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+
"\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+
"\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+
"\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+
"\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+
"\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+
"\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+
"@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+
"\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+
"\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+
"\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+
"$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+
"\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+
"*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+
"-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+
"01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+
"\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+
"\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+
"i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+
"m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+
"\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+
"\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+
"\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+
"\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+
"KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+
"MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+
"\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+
"\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+
"\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+
"\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+
"YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+
"\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+
"\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+
"_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+
"\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
"\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
"fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+
"\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
public static final ATN _ATN =
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
static {
Expand Down
2 changes: 1 addition & 1 deletion src/main/grammars/Aggregation.g4
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ RPAREN : ')' ;

DECIMAL : '-'?[0-9]+('.'[0-9]+)? ;

IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ;
IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ;
WS : [ \r\t\u000C\n]+ -> skip ;

comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,9 @@

package org.opensearch.securityanalytics.mapper;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.action.ActionListener;
import org.opensearch.action.admin.indices.get.GetIndexRequest;
import org.opensearch.action.admin.indices.get.GetIndexResponse;
Expand All @@ -44,6 +33,16 @@
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.mapper.MapperUtils.PATH;
import static org.opensearch.securityanalytics.mapper.MapperUtils.PROPERTIES;
Expand Down Expand Up @@ -78,9 +77,11 @@ public void createMappingAction(String indexName, String logType, String aliasMa
// since you can't update documents in non-write indices
String index = indexName;
boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
if (writeIndex != null) {
log.debug("Write index for {} is {}", indexName, writeIndex);
index = writeIndex;
}
}
Expand All @@ -92,6 +93,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
applyAliasMappings(getMappingsResponse.getMappings(), logType, aliasMappings, partial, new ActionListener<>() {
@Override
public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
log.debug("Completed create mappings for {}", indexName);
// We will return ack==false if one of the requests returned that
// else return ack==true
Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
Expand All @@ -110,6 +112,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {

@Override
public void onFailure(Exception e) {
log.debug("Failed to create mappings for {}", indexName );
actionListener.onFailure(e);
}
});
Expand Down Expand Up @@ -478,13 +481,16 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
String rawPath = requiredField.getRawField();
String ocsfPath = requiredField.getOcsf();
if (allFieldsFromIndex.contains(rawPath)) {
if (alias != null) {
// Maintain list of found paths in index
applyableAliases.add(alias);
} else {
applyableAliases.add(rawPath);
// if the alias was already added into applyable aliases, then skip to avoid duplicates
if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) {
if (alias != null) {
// Maintain list of found paths in index
applyableAliases.add(alias);
} else {
applyableAliases.add(rawPath);
}
pathsOfApplyableAliases.add(rawPath);
}
pathsOfApplyableAliases.add(rawPath);
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
Expand All @@ -498,13 +504,23 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
}
}

// turn unmappedFieldAliases into a set to remove duplicates
Set<String> setOfUnmappedFieldAliases = new HashSet<>(unmappedFieldAliases);

// filter out aliases that were included in applyableAliases already
List<String> filteredUnmappedFieldAliases = setOfUnmappedFieldAliases.stream()
.filter(e -> false == applyableAliases.contains(e))
.collect(Collectors.toList());

Map<String, Map<String, String>> aliasMappingFields = new HashMap<>();
XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject();
for (LogType.Mapping mapping: requiredFields) {
if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (mapping.getEcs() != null) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
} else if (mapping.getEcs() == null) {
aliasMappingFields.put(mapping.getRawField(), Map.of("type", "alias", "path", mapping.getRawField()));
}
}
aliasMappingsObj.field("properties", aliasMappingFields);
Expand All @@ -519,7 +535,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
.collect(Collectors.toList());

actionListener.onResponse(
new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases)
new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases)
);
} catch (Exception e) {
actionListener.onFailure(e);
Expand All @@ -533,6 +549,26 @@ public void onFailure(Exception e) {
});
}

/**
* Only updates the alias mapping fields if the ecs key has not been mapped yet
* or if pathOfApplyableAliases contains the raw field
*
* @param mapping
* @param aliasMappingFields
* @param pathsOfApplyableAliases
*/
private static void shouldUpdateEcsMappingAndMaybeUpdates(LogType.Mapping mapping, Map<String, Map<String, String>> aliasMappingFields, List<String> pathsOfApplyableAliases) {
// check if aliasMappingFields already contains a key
if (aliasMappingFields.containsKey(mapping.getEcs())) {
// if the pathOfApplyableAliases contains the raw field, then override the existing map
if (pathsOfApplyableAliases.contains(mapping.getRawField())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
}
} else {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
}
}

/**
* Given index name, resolves it to single concrete index, depending on what initial <code>indexName</code> is.
* In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@

package org.opensearch.securityanalytics.mapper;

import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

public class MapperUtils {

Expand Down Expand Up @@ -246,7 +247,6 @@ public void onError(String error) {
}
});
mappingsTraverser.traverse();

return presentPathsMappings;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat
aggregationBuilder = new TermsAggregationBuilder(name).field(name);
break;
case "count":
aggregationBuilder = new ValueCountAggregationBuilder(name).field(name);
aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name);
break;
default:
throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction));
Expand Down
Loading

0 comments on commit e44e5d4

Please sign in to comment.