Skip to content

Commit

Permalink
Merge branch 'main' into add_exists_check
Browse files Browse the repository at this point in the history 
Signed-off-by: Joanne Wang <jowg@amazon.com>
  • Loading branch information
@jowg-amazon
jowg-amazon committed Mar 7, 2024
2 parents e774d02 + 689760e commit c71312a
Show file tree
Hide file tree
Showing 23 changed files with 1,717 additions and 1,700 deletions.
541 changes: 242 additions & 299 deletions src/main/java/org/opensearch/securityanalytics/correlation/JoinEngine.java
View file

Large diffs are not rendered by default.

813 changes: 345 additions & 468 deletions src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ public void onResponse(SearchResponse response) {
if (response.isTimedOut()) {
listener.onFailure(new OpenSearchStatusException("Search request timed out", RestStatus.REQUEST_TIMEOUT));
}
if (response.getHits().getTotalHits().value > 0) {
if (response.getHits().getHits().length > 0) {
listener.onResponse(null);
} else {
try {
Expand Down
54 changes: 44 additions & 10 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.OpenSearchStatusException;
import org.opensearch.action.admin.indices.get.GetIndexRequest;
import org.opensearch.action.admin.indices.get.GetIndexResponse;
import org.opensearch.action.admin.indices.mapping.get.GetMappingsRequest;
Expand Down Expand Up @@ -78,9 +77,11 @@ public void createMappingAction(String indexName, String logType, String aliasMa
// since you can't update documents in non-write indices
String index = indexName;
boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
if (writeIndex != null) {
log.debug("Write index for {} is {}", indexName, writeIndex);
index = writeIndex;
}
}
Expand All @@ -92,6 +93,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
applyAliasMappings(getMappingsResponse.getMappings(), logType, aliasMappings, partial, new ActionListener<>() {
@Override
public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
log.debug("Completed create mappings for {}", indexName);
// We will return ack==false if one of the requests returned that
// else return ack==true
Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
Expand All @@ -110,6 +112,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {

@Override
public void onFailure(Exception e) {
log.debug("Failed to create mappings for {}", indexName );
actionListener.onFailure(e);
}
});
Expand Down Expand Up @@ -481,13 +484,16 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
String rawPath = requiredField.getRawField();
String ocsfPath = requiredField.getOcsf();
if (allFieldsFromIndex.contains(rawPath)) {
if (alias != null) {
// Maintain list of found paths in index
applyableAliases.add(alias);
} else {
applyableAliases.add(rawPath);
// if the alias was already added into applyable aliases, then skip to avoid duplicates
if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) {
if (alias != null) {
// Maintain list of found paths in index
applyableAliases.add(alias);
} else {
applyableAliases.add(rawPath);
}
pathsOfApplyableAliases.add(rawPath);
}
pathsOfApplyableAliases.add(rawPath);
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
Expand All @@ -501,13 +507,21 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
}
}

// turn unmappedFieldAliases into a set to remove duplicates
Set<String> setOfUnmappedFieldAliases = new HashSet<>(unmappedFieldAliases);

// filter out aliases that were included in applyableAliases already
List<String> filteredUnmappedFieldAliases = setOfUnmappedFieldAliases.stream()
.filter(e -> false == applyableAliases.contains(e))
.collect(Collectors.toList());

Map<String, Map<String, String>> aliasMappingFields = new HashMap<>();
XContentBuilder aliasMappingsObj = XContentFactory.jsonBuilder().startObject();
for (LogType.Mapping mapping : requiredFields) {
if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (mapping.getEcs() != null) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
} else if (mapping.getEcs() == null) {
aliasMappingFields.put(mapping.getRawField(), Map.of("type", "alias", "path", mapping.getRawField()));
}
Expand All @@ -523,7 +537,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
.filter(e -> pathsOfApplyableAliases.contains(e) == false)
.collect(Collectors.toList());
actionListener.onResponse(
new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, unmappedFieldAliases, logTypeService.getIocFieldsList(logType))
new GetMappingsViewResponse(aliasMappings, unmappedIndexFields, filteredUnmappedFieldAliases, logTypeService.getIocFieldsList(logType))
);
} catch (Exception e) {
actionListener.onFailure(e);
Expand All @@ -538,6 +552,26 @@ public void onFailure(Exception e) {
});
}

/**
* Only updates the alias mapping fields if the ecs key has not been mapped yet
* or if pathOfApplyableAliases contains the raw field
*
* @param mapping
* @param aliasMappingFields
* @param pathsOfApplyableAliases
*/
private static void shouldUpdateEcsMappingAndMaybeUpdates(LogType.Mapping mapping, Map<String, Map<String, String>> aliasMappingFields, List<String> pathsOfApplyableAliases) {
// check if aliasMappingFields already contains a key
if (aliasMappingFields.containsKey(mapping.getEcs())) {
// if the pathOfApplyableAliases contains the raw field, then override the existing map
if (pathsOfApplyableAliases.contains(mapping.getRawField())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
}
} else {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
}
}

/**
* Given index name, resolves it to single concrete index, depending on what initial <code>indexName</code> is.
* In case of Datastream or Alias, WriteIndex would be returned. In case of index pattern, newest index by creation date would be returned.
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@

package org.opensearch.securityanalytics.mapper;

import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;

public class MapperUtils {

Expand Down Expand Up @@ -246,7 +247,6 @@ public void onError(String error) {
}
});
mappingsTraverser.traverse();

return presentPathsMappings;
}
}
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -383,11 +383,14 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre

@Override
public Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError {
String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
SigmaString value = (SigmaString) condition.getValue();
boolean containsWildcard = value.containsWildcard();
String exprWithDeMorgansApplied = this.notToken + " " + "%s";

String conditionValStr = String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));
String conditionValStr = String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression),
this.convertValueStr((SigmaString) condition.getValue()));
if (applyDeMorgans) {
conditionValStr = String.format(Locale.getDefault(), exprWithDeMorgansApplied, conditionValStr);
}
Expand Down
10 changes: 9 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/rules/objects/SigmaDetectionItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.opensearch.securityanalytics.rules.modifiers.SigmaModifierFacade;
import org.opensearch.securityanalytics.rules.modifiers.SigmaValueModifier;
import org.opensearch.securityanalytics.rules.types.SigmaNull;
import org.opensearch.securityanalytics.rules.types.SigmaString;
import org.opensearch.securityanalytics.rules.types.SigmaType;
import org.opensearch.securityanalytics.rules.types.SigmaTypeFacade;
import org.opensearch.securityanalytics.rules.utils.AnyOneOf;
Expand Down Expand Up @@ -111,7 +112,14 @@ public static <T> SigmaDetectionItem fromMapping(String key, Either<T, List<T>>

List<SigmaType> sigmaTypes = new ArrayList<>();
for (T v: values) {
sigmaTypes.add(SigmaTypeFacade.sigmaType(v));
SigmaType sigmaType = SigmaTypeFacade.sigmaType(v);
// throws an error if sigmaType is an empty string and the modifier is "contains" or "startswith" or "endswith"
boolean invalidModifierWithEmptyString = modifierIds.contains("contains") || modifierIds.contains("startswith") || modifierIds.contains("endswith");
if (sigmaType.getClass().equals(SigmaString.class) && v.toString().isEmpty() && invalidModifierWithEmptyString) {
throw new SigmaValueError("Cannot create rule with empty string and given modifier(s): " + modifierIds);
} else {
sigmaTypes.add(sigmaType);
}
}

return new SigmaDetectionItem(field, modifiers, sigmaTypes, null, null, true);
Expand Down
43 changes: 15 additions & 28 deletions src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.model.Detector.DETECTORS_INDEX;
Expand Down Expand Up @@ -121,35 +119,24 @@ public void createDocLevelQueryFromThreatIntel(List<LogType.IocFields> iocFieldL
listener.onResponse(Collections.emptyList());
return;
}

CountDownLatch latch = new CountDownLatch(1);
threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() {
@Override
public void onResponse(List<ThreatIntelFeedData> threatIntelFeedData) {
if (threatIntelFeedData.isEmpty()) {
listener.onResponse(Collections.emptyList());
} else {
listener.onResponse(
createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
);
threatIntelFeedDataService.getThreatIntelFeedData(ActionListener.wrap(
threatIntelFeedData -> {
if (threatIntelFeedData.isEmpty()) {
listener.onResponse(Collections.emptyList());
} else {
listener.onResponse(
createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
);
}
}, e -> {
log.error("Failed to get threat intel feeds for doc level query creation", e);
listener.onFailure(e);
}
latch.countDown();
}

@Override
public void onFailure(Exception e) {
log.error("Failed to get threat intel feeds for doc level query creation", e);
listener.onFailure(e);
latch.countDown();
}
});

latch.await(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
log.error("Failed to create doc level queries from threat intel feeds", e);
));
} catch (Exception e) {
log.error("Failed to create doc level query from threat intel data", e);
listener.onFailure(e);
}

}

private static String constructId(Detector detector, String iocType) {
Expand Down
Loading

0 comments on commit c71312a

Please sign in to comment.