Skip to content

Commit

Permalink
Add custom result index lifecycle management condition fields to conf…
Browse files Browse the repository at this point in the history 
…ig (#1215)

* Add custom result index lifecycle management condition fields to config

Signed-off-by: Jackie Han <jkhanjob@gmail.com>

* add more tests

Signed-off-by: Jackie Han <jkhanjob@gmail.com>

* utilizing readOptionalInt() method when reading nullable value

Signed-off-by: Jackie Han <jkhanjob@gmail.com>

* add more tests

Signed-off-by: Jackie Han <jkhanjob@gmail.com>

---------

Signed-off-by: Jackie Han <jkhanjob@gmail.com>
  • Loading branch information
@jackiehanyang
jackiehanyang committed Jun 3, 2024
1 parent 2ce5a7c commit 05b04b9
Show file tree
Hide file tree
Showing 18 changed files with 403 additions and 30 deletions.
6 changes: 4 additions & 2 deletions src/main/java/org/opensearch/ad/model/ADTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,8 +341,10 @@ public static ADTask parse(XContentParser parser, String taskId) throws IOExcept
detector.getRecencyEmphasis(),
detector.getSeasonIntervals(),
detector.getHistoryIntervals(),
detector.getRules()

detector.getRules(),
detector.getCustomResultIndexMinSize(),
detector.getCustomResultIndexMinAge(),
detector.getCustomResultIndexTTL()
);
return new Builder()
.taskId(parsedTaskId)
Expand Down
39 changes: 34 additions & 5 deletions src/main/java/org/opensearch/ad/model/AnomalyDetector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ public Integer getShingleSize(Integer customShingleSize) {
* @param seasonIntervals seasonality in terms of intervals
* @param historyIntervals history intervals we look back during cold start
* @param rules custom rules to filter out AD results
* @param customResultIndexMinSize custom result index lifecycle management min size condition
* @param customResultIndexMinAge custom result index lifecycle management min age condition
* @param customResultIndexTTL custom result index lifecycle management ttl
*/
public AnomalyDetector(
String detectorId,
Expand All @@ -170,7 +173,10 @@ public AnomalyDetector(
Integer recencyEmphasis,
Integer seasonIntervals,
Integer historyIntervals,
List<Rule> rules
List<Rule> rules,
Integer customResultIndexMinSize,
Integer customResultIndexMinAge,
Integer customResultIndexTTL
) {
super(
detectorId,
Expand All @@ -194,7 +200,10 @@ public AnomalyDetector(
recencyEmphasis,
seasonIntervals,
new ADShingleGetter(seasonIntervals),
historyIntervals
historyIntervals,
customResultIndexMinSize,
customResultIndexMinAge,
customResultIndexTTL
);

checkAndThrowValidationErrors(ValidationAspect.DETECTOR);
Expand Down Expand Up @@ -268,14 +277,17 @@ public AnomalyDetector(StreamInput input) throws IOException {
if (input.readBoolean()) {
this.rules = input.readList(Rule::new);
}
this.customResultIndexMinSize = input.readOptionalInt();
this.customResultIndexMinAge = input.readOptionalInt();
this.customResultIndexTTL = input.readOptionalInt();
}

public XContentBuilder toXContent(XContentBuilder builder) throws IOException {
return toXContent(builder, ToXContent.EMPTY_PARAMS);
}

/*
* For backward compatiblity reason, we cannot use super class
* For backward compatibility reason, we cannot use super class
* Config's writeTo as we have detectionDateRange and
* detectorType that Config does not have.
*/
Expand Down Expand Up @@ -330,6 +342,9 @@ public void writeTo(StreamOutput output) throws IOException {
} else {
output.writeBoolean(false);
}
output.writeOptionalInt(customResultIndexMinSize);
output.writeOptionalInt(customResultIndexMinAge);
output.writeOptionalInt(customResultIndexTTL);
}

@Override
Expand Down Expand Up @@ -422,8 +437,10 @@ public static AnomalyDetector parse(
Integer recencyEmphasis = null;
Integer seasonality = null;
Integer historyIntervals = null;

List<Rule> rules = new ArrayList<>();
Integer customResultIndexMinSize = null;
Integer customResultIndexMinAge = null;
Integer customResultIndexTTL = null;

ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
Expand Down Expand Up @@ -549,6 +566,15 @@ public static AnomalyDetector parse(
rules.add(Rule.parse(parser));
}
break;
case RESULT_INDEX_FIELD_MIN_SIZE:
customResultIndexMinSize = parser.intValue();
break;
case RESULT_INDEX_FIELD_MIN_AGE:
customResultIndexMinAge = parser.intValue();
break;
case RESULT_INDEX_FIELD_TTL:
customResultIndexTTL = parser.intValue();
break;
default:
parser.skipChildren();
break;
Expand Down Expand Up @@ -576,7 +602,10 @@ public static AnomalyDetector parse(
recencyEmphasis,
seasonality,
historyIntervals,
rules
rules,
customResultIndexMinSize,
customResultIndexMinAge,
customResultIndexTTL
);
detector.setDetectionDateRange(detectionDateRange);
return detector;
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,10 @@ protected AnomalyDetector copyConfig(User user, Config config) {
config.getRecencyEmphasis(),
config.getSeasonIntervals(),
config.getHistoryIntervals(),
detector.getRules()
detector.getRules(),
config.getCustomResultIndexMinSize(),
config.getCustomResultIndexMinAge(),
config.getCustomResultIndexTTL()
);
}

Expand Down
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/forecast/model/ForecastTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,10 @@ public static ForecastTask parse(XContentParser parser, String taskId) throws IO
forecaster.getImputationOption(),
forecaster.getRecencyEmphasis(),
forecaster.getSeasonIntervals(),
forecaster.getHistoryIntervals()
forecaster.getHistoryIntervals(),
forecaster.getCustomResultIndexMinSize(),
forecaster.getCustomResultIndexMinAge(),
forecaster.getCustomResultIndexTTL()
);
return new Builder()
.taskId(parsedTaskId)
Expand Down
27 changes: 24 additions & 3 deletions src/main/java/org/opensearch/forecast/model/Forecaster.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,10 @@ public Forecaster(
ImputationOption imputationOption,
Integer recencyEmphasis,
Integer seasonIntervals,
Integer historyIntervals
Integer historyIntervals,
Integer customResultIndexMinSize,
Integer customResultIndexMinAge,
Integer customResultIndexTTL
) {
super(
forecasterId,
Expand All @@ -155,7 +158,10 @@ public Forecaster(
recencyEmphasis,
seasonIntervals,
new ForecastShingleGetter(seasonIntervals, horizon),
historyIntervals
historyIntervals,
customResultIndexMinSize,
customResultIndexMinAge,
customResultIndexTTL
);

checkAndThrowValidationErrors(ValidationAspect.FORECASTER);
Expand Down Expand Up @@ -294,6 +300,9 @@ public static Forecaster parse(
Integer recencyEmphasis = null;
Integer seasonality = null;
Integer historyIntervals = null;
Integer customResultIndexMinSize = null;
Integer customResultIndexMinAge = null;
Integer customResultIndexTTL = null;

ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
Expand Down Expand Up @@ -413,6 +422,15 @@ public static Forecaster parse(
case HISTORY_INTERVAL_FIELD:
historyIntervals = parser.intValue();
break;
case RESULT_INDEX_FIELD_MIN_SIZE:
customResultIndexMinSize = parser.intValue();
break;
case RESULT_INDEX_FIELD_MIN_AGE:
customResultIndexMinAge = parser.intValue();
break;
case RESULT_INDEX_FIELD_TTL:
customResultIndexTTL = parser.intValue();
break;
default:
parser.skipChildren();
break;
Expand Down Expand Up @@ -440,7 +458,10 @@ public static Forecaster parse(
imputationOption,
recencyEmphasis,
seasonality,
historyIntervals
historyIntervals,
customResultIndexMinSize,
customResultIndexMinAge,
customResultIndexTTL
);
return forecaster;
}
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/org/opensearch/forecast/rest/handler/AbstractForecasterActionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,10 @@ protected Config copyConfig(User user, Config config) {
config.getImputationOption(),
config.getRecencyEmphasis(),
config.getSeasonIntervals(),
config.getHistoryIntervals()
config.getHistoryIntervals(),
config.getCustomResultIndexMinSize(),
config.getCustomResultIndexMinAge(),
config.getCustomResultIndexTTL()
);
}

Expand Down
54 changes: 51 additions & 3 deletions src/main/java/org/opensearch/timeseries/model/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ public abstract class Config implements Writeable, ToXContentObject {
public static final String SEASONALITY_FIELD = "suggested_seasonality";
public static final String RECENCY_EMPHASIS_FIELD = "recency_emphasis";
public static final String HISTORY_INTERVAL_FIELD = "history";
public static final String RESULT_INDEX_FIELD_MIN_SIZE = "result_index_min_size";
public static final String RESULT_INDEX_FIELD_MIN_AGE = "result_index_min_age";
public static final String RESULT_INDEX_FIELD_TTL = "result_index_ttl";

protected String id;
protected Long version;
Expand Down Expand Up @@ -111,6 +114,9 @@ public abstract class Config implements Writeable, ToXContentObject {

protected Integer seasonIntervals;
protected Integer historyIntervals;
protected Integer customResultIndexMinSize;
protected Integer customResultIndexMinAge;
protected Integer customResultIndexTTL;

public static String INVALID_RESULT_INDEX_NAME_SIZE = "Result index name size must contains less than "
+ MAX_RESULT_INDEX_NAME_SIZE
Expand Down Expand Up @@ -138,7 +144,10 @@ protected Config(
Integer recencyEmphasis,
Integer seasonIntervals,
ShingleGetter shingleGetter,
Integer historyIntervals
Integer historyIntervals,
Integer customResultIndexMinSize,
Integer customResultIndexMinAge,
Integer customResultIndexTTL
) {
if (Strings.isBlank(name)) {
errorMessage = CommonMessages.EMPTY_NAME;
Expand Down Expand Up @@ -254,6 +263,9 @@ protected Config(
this.recencyEmphasis = Optional.ofNullable(recencyEmphasis).orElse(TimeSeriesSettings.DEFAULT_RECENCY_EMPHASIS);
this.seasonIntervals = seasonIntervals;
this.historyIntervals = historyIntervals == null ? suggestHistory() : historyIntervals;
this.customResultIndexMinSize = Strings.trimToNull(resultIndex) == null ? null : customResultIndexMinSize;
this.customResultIndexMinAge = Strings.trimToNull(resultIndex) == null ? null : customResultIndexMinAge;
this.customResultIndexTTL = Strings.trimToNull(resultIndex) == null ? null : customResultIndexTTL;
}

public int suggestHistory() {
Expand Down Expand Up @@ -294,6 +306,9 @@ public Config(StreamInput input) throws IOException {
this.recencyEmphasis = input.readInt();
this.seasonIntervals = input.readInt();
this.historyIntervals = input.readInt();
this.customResultIndexMinSize = input.readOptionalInt();
this.customResultIndexMinAge = input.readOptionalInt();
this.customResultIndexTTL = input.readOptionalInt();
}

/*
Expand Down Expand Up @@ -343,6 +358,9 @@ public void writeTo(StreamOutput output) throws IOException {
output.writeInt(recencyEmphasis);
output.writeInt(seasonIntervals);
output.writeInt(historyIntervals);
output.writeOptionalInt(customResultIndexMinSize);
output.writeOptionalInt(customResultIndexMinAge);
output.writeOptionalInt(customResultIndexTTL);
}

public boolean invalidShingleSizeRange(Integer shingleSizeToTest) {
Expand Down Expand Up @@ -396,7 +414,10 @@ public boolean equals(Object o) {
&& Objects.equal(imputationOption, config.imputationOption)
&& Objects.equal(recencyEmphasis, config.recencyEmphasis)
&& Objects.equal(seasonIntervals, config.seasonIntervals)
&& Objects.equal(historyIntervals, config.historyIntervals);
&& Objects.equal(historyIntervals, config.historyIntervals)
&& Objects.equal(customResultIndexMinSize, config.customResultIndexMinSize)
&& Objects.equal(customResultIndexMinAge, config.customResultIndexMinAge)
&& Objects.equal(customResultIndexTTL, config.customResultIndexTTL);
}

@Generated
Expand All @@ -420,7 +441,10 @@ public int hashCode() {
imputationOption,
recencyEmphasis,
seasonIntervals,
historyIntervals
historyIntervals,
customResultIndexMinSize,
customResultIndexMinAge,
customResultIndexTTL
);
}

Expand Down Expand Up @@ -460,6 +484,15 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (seasonIntervals != null) {
builder.field(SEASONALITY_FIELD, seasonIntervals);
}
if (customResultIndexMinSize != null) {
builder.field(RESULT_INDEX_FIELD_MIN_SIZE, customResultIndexMinSize);
}
if (customResultIndexMinAge != null) {
builder.field(RESULT_INDEX_FIELD_MIN_AGE, customResultIndexMinAge);
}
if (customResultIndexTTL != null) {
builder.field(RESULT_INDEX_FIELD_TTL, customResultIndexTTL);
}
return builder;
}

Expand Down Expand Up @@ -648,6 +681,18 @@ public Integer getHistoryIntervals() {
return historyIntervals;
}

public Integer getCustomResultIndexMinSize() {
return customResultIndexMinSize;
}

public Integer getCustomResultIndexMinAge() {
return customResultIndexMinAge;
}

public Integer getCustomResultIndexTTL() {
return customResultIndexTTL;
}

/**
* Identifies redundant feature names.
*
Expand Down Expand Up @@ -699,6 +744,9 @@ public String toString() {
.append("recencyEmphasis", recencyEmphasis)
.append("seasonIntervals", seasonIntervals)
.append("historyIntervals", historyIntervals)
.append("customResultIndexMinSize", customResultIndexMinSize)
.append("customResultIndexMinAge", customResultIndexMinAge)
.append("customResultIndexTTL", customResultIndexTTL)
.toString();
}
}
10 changes: 8 additions & 2 deletions src/test/java/org/opensearch/ad/AnomalyDetectorRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,10 @@ public ToXContentObject[] getConfig(String detectorId, BasicHeader header, boole
detector.getRecencyEmphasis(),
detector.getSeasonIntervals(),
detector.getHistoryIntervals(),
null
null,
detector.getCustomResultIndexMinSize(),
detector.getCustomResultIndexMinAge(),
detector.getCustomResultIndexTTL()
),
detectorJob,
historicalAdTask,
Expand Down Expand Up @@ -647,7 +650,10 @@ protected AnomalyDetector cloneDetector(AnomalyDetector anomalyDetector, String
anomalyDetector.getRecencyEmphasis(),
anomalyDetector.getSeasonIntervals(),
anomalyDetector.getHistoryIntervals(),
null
null,
anomalyDetector.getCustomResultIndexMinSize(),
anomalyDetector.getCustomResultIndexMinAge(),
anomalyDetector.getCustomResultIndexTTL()
);
return detector;
}
Expand Down
Loading

0 comments on commit 05b04b9

Please sign in to comment.