Skip to content

Commit

Permalink
Add max_single_primary_size as a condition for the ILM rollover action (
Browse files Browse the repository at this point in the history 
#68917) (#69202)
  • Loading branch information
@joegallo
joegallo committed Feb 18, 2021
1 parent 3250dd7 commit 128f205
Show file tree
Hide file tree
Showing 21 changed files with 249 additions and 120 deletions.
35 changes: 26 additions & 9 deletions ...rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,43 +20,56 @@
import java.io.IOException;
import java.util.Objects;


public class RolloverAction implements LifecycleAction, ToXContentObject {
public static final String NAME = "rollover";
private static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
private static final ParseField MAX_SINGLE_PRIMARY_SIZE_FIELD = new ParseField("max_single_primary_size");
private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");

private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
a -> new RolloverAction((ByteSizeValue) a[0], (ByteSizeValue) a[1], (TimeValue) a[2], (Long) a[3]));

static {
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()),
MAX_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName()),
MAX_SINGLE_PRIMARY_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()),
MAX_AGE_FIELD, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
}

private final ByteSizeValue maxSize;
private final Long maxDocs;
private final ByteSizeValue maxSinglePrimarySize;
private final TimeValue maxAge;
private final Long maxDocs;

public static RolloverAction parse(XContentParser parser) {
return PARSER.apply(parser, null);
}

public RolloverAction(ByteSizeValue maxSize, TimeValue maxAge, Long maxDocs) {
if (maxSize == null && maxAge == null && maxDocs == null) {
public RolloverAction(ByteSizeValue maxSize, ByteSizeValue maxSinglePrimarySize, TimeValue maxAge, Long maxDocs) {
if (maxSize == null && maxSinglePrimarySize == null && maxAge == null && maxDocs == null) {
throw new IllegalArgumentException("At least one rollover condition must be set.");
}
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}

public ByteSizeValue getMaxSize() {
return maxSize;
}

public ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

public TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -76,6 +89,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (maxSize != null) {
builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
}
if (maxSinglePrimarySize != null) {
builder.field(MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName(), maxSinglePrimarySize.getStringRep());
}
if (maxAge != null) {
builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
}
Expand All @@ -88,7 +104,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws

@Override
public int hashCode() {
return Objects.hash(maxSize, maxAge, maxDocs);
return Objects.hash(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -101,6 +117,7 @@ public boolean equals(Object obj) {
}
RolloverAction other = (RolloverAction) obj;
return Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}
Expand Down
2 changes: 1 addition & 1 deletion client/rest-high-level/src/test/java/org/elasticsearch/client/IndexLifecycleIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ public void testStartStopILM() throws Exception {
public void testExplainLifecycle() throws Exception {
Map<String, Phase> lifecyclePhases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(null, TimeValue.timeValueHours(50 * 24), null));
hotActions.put(RolloverAction.NAME, new RolloverAction(null, null, TimeValue.timeValueHours(50 * 24), null));
Phase hotPhase = new Phase("hot", randomFrom(TimeValue.ZERO, null), hotActions);
lifecyclePhases.put("hot", hotPhase);

Expand Down
8 changes: 4 additions & 4 deletions ...t-high-level/src/test/java/org/elasticsearch/client/documentation/ILMDocumentationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void testPutLifecyclePolicy() throws Exception {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions)); // <1>

Map<String, LifecycleAction> deleteActions =
Expand Down Expand Up @@ -164,7 +164,7 @@ public void testDeletePolicy() throws IOException, InterruptedException {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));
Map<String, LifecycleAction> deleteActions =
Collections.singletonMap(DeleteAction.NAME,
Expand Down Expand Up @@ -239,7 +239,7 @@ public void testGetLifecyclePolicy() throws IOException, InterruptedException {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));

Map<String, LifecycleAction> deleteActions =
Expand Down Expand Up @@ -339,7 +339,7 @@ public void testExplainLifecycle() throws Exception {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> hotActions = new HashMap<>();
hotActions.put(RolloverAction.NAME, new RolloverAction(
new ByteSizeValue(50, ByteSizeUnit.GB), null, null));
new ByteSizeValue(50, ByteSizeUnit.GB), null, null, null));
phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions));

LifecyclePolicy policy = new LifecyclePolicy("my_policy",
Expand Down
18 changes: 11 additions & 7 deletions ...high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RolloverActionTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,21 @@ protected RolloverAction createTestInstance() {

static RolloverAction randomInstance() {
ByteSizeUnit maxSizeUnit = randomFrom(ByteSizeUnit.values());
ByteSizeValue maxSize = randomBoolean() ? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
Long maxDocs = randomBoolean() ? null : randomNonNegativeLong();
TimeValue maxAge = (maxDocs == null && maxSize == null || randomBoolean())
? TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test")
: null;
return new RolloverAction(maxSize, maxAge, maxDocs);
ByteSizeValue maxSize = randomBoolean()
? null : new ByteSizeValue(randomNonNegativeLong() / maxSizeUnit.toBytes(1), maxSizeUnit);
ByteSizeUnit maxSinglePrimarySizeUnit = randomFrom(ByteSizeUnit.values());
ByteSizeValue maxSinglePrimarySize = randomBoolean()
? null : new ByteSizeValue(randomNonNegativeLong() / maxSinglePrimarySizeUnit.toBytes(1), maxSinglePrimarySizeUnit);
TimeValue maxAge = randomBoolean()
? null : TimeValue.parseTimeValue(randomPositiveTimeValue(), "rollover_action_test");
Long maxDocs = (maxSize == null && maxSinglePrimarySize == null && maxAge == null || randomBoolean())
? randomNonNegativeLong() : null;
return new RolloverAction(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

public void testNoConditions() {
IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
() -> new RolloverAction(null, null, null));
() -> new RolloverAction(null, null, null, null));
assertEquals("At least one rollover condition must be set.", exception.getMessage());
}
}
10 changes: 10 additions & 0 deletions docs/reference/ilm/actions/ilm-rollover.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,16 @@ Replicas are not counted toward the maximum index size.
TIP: To see the current index size, use the <<cat-indices, _cat indices>> API.
The `pri.store.size` value shows the combined size of all primary shards.

`max_single_primary_size`::
(Optional, <<byte-units, byte units>>)
Triggers roll over when the largest primary shard in the index reaches a certain size.
This is the maximum size of the primary shards in the index. As with `max_size`,
replicas are ignored.
+
TIP: To see the current shard size, use the <<cat-shards, _cat shards>> API.
The `store` value shows the size each shard, and `prirep` indicates whether a
shard is a primary (`p`) or a replica (`r`).

[[ilm-rollover-ex]]
==== Example

Expand Down
59 changes: 41 additions & 18 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*/
package org.elasticsearch.xpack.core.ilm;

import org.elasticsearch.Version;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
Expand Down Expand Up @@ -34,35 +35,45 @@ public class RolloverAction implements LifecycleAction {
public static final String NAME = "rollover";
public static final String INDEXING_COMPLETE_STEP_NAME = "set-indexing-complete";
public static final ParseField MAX_SIZE_FIELD = new ParseField("max_size");
public static final ParseField MAX_SINGLE_PRIMARY_SIZE_FIELD = new ParseField("max_single_primary_size");
public static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
public static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
public static final String LIFECYCLE_ROLLOVER_ALIAS = "index.lifecycle.rollover_alias";
public static final Setting<String> LIFECYCLE_ROLLOVER_ALIAS_SETTING = Setting.simpleString(LIFECYCLE_ROLLOVER_ALIAS,
Setting.Property.Dynamic, Setting.Property.IndexScope);

private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
a -> new RolloverAction((ByteSizeValue) a[0], (ByteSizeValue) a[1], (TimeValue) a[2], (Long) a[3]));

static {
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()), MAX_SIZE_FIELD, ValueType.VALUE);
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SIZE_FIELD.getPreferredName()),
MAX_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName()),
MAX_SINGLE_PRIMARY_SIZE_FIELD, ValueType.VALUE);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()), MAX_AGE_FIELD, ValueType.VALUE);
(p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE_FIELD.getPreferredName()),
MAX_AGE_FIELD, ValueType.VALUE);
PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MAX_DOCS_FIELD);
}

private final ByteSizeValue maxSize;
private final ByteSizeValue maxSinglePrimarySize;
private final Long maxDocs;
private final TimeValue maxAge;

public static RolloverAction parse(XContentParser parser) {
return PARSER.apply(parser, null);
}

public RolloverAction(@Nullable ByteSizeValue maxSize, @Nullable TimeValue maxAge, @Nullable Long maxDocs) {
if (maxSize == null && maxAge == null && maxDocs == null) {
public RolloverAction(@Nullable ByteSizeValue maxSize, @Nullable ByteSizeValue maxSinglePrimarySize, @Nullable TimeValue maxAge,
@Nullable Long maxDocs) {
if (maxSize == null && maxSinglePrimarySize == null && maxAge == null && maxDocs == null) {
throw new IllegalArgumentException("At least one rollover condition must be set.");
}
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}
Expand All @@ -73,12 +84,13 @@ public RolloverAction(StreamInput in) throws IOException {
} else {
maxSize = null;
}
maxAge = in.readOptionalTimeValue();
if (in.readBoolean()) {
maxDocs = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_7_13_0) && in.readBoolean()) {
maxSinglePrimarySize = new ByteSizeValue(in);
} else {
maxDocs = null;
maxSinglePrimarySize = null;
}
maxAge = in.readOptionalTimeValue();
maxDocs = in.readOptionalVLong();
}

@Override
Expand All @@ -88,12 +100,15 @@ public void writeTo(StreamOutput out) throws IOException {
if (hasMaxSize) {
maxSize.writeTo(out);
}
out.writeOptionalTimeValue(maxAge);
boolean hasMaxDocs = maxDocs != null;
out.writeBoolean(hasMaxDocs);
if (hasMaxDocs) {
out.writeVLong(maxDocs);
if (out.getVersion().onOrAfter(Version.V_7_13_0)) {
boolean hasMaxSinglePrimarySize = maxSinglePrimarySize != null;
out.writeBoolean(hasMaxSinglePrimarySize);
if (hasMaxSinglePrimarySize) {
maxSinglePrimarySize.writeTo(out);
}
}
out.writeOptionalTimeValue(maxAge);
out.writeOptionalVLong(maxDocs);
}

@Override
Expand All @@ -105,6 +120,10 @@ public ByteSizeValue getMaxSize() {
return maxSize;
}

public ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

public TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -119,6 +138,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (maxSize != null) {
builder.field(MAX_SIZE_FIELD.getPreferredName(), maxSize.getStringRep());
}
if (maxSinglePrimarySize != null) {
builder.field(MAX_SINGLE_PRIMARY_SIZE_FIELD.getPreferredName(), maxSinglePrimarySize.getStringRep());
}
if (maxAge != null) {
builder.field(MAX_AGE_FIELD.getPreferredName(), maxAge.getStringRep());
}
Expand All @@ -145,7 +167,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)
StepKey setIndexingCompleteStepKey = new StepKey(phase, NAME, INDEXING_COMPLETE_STEP_NAME);

WaitForRolloverReadyStep waitForRolloverReadyStep = new WaitForRolloverReadyStep(waitForRolloverReadyStepKey, rolloverStepKey,
client, maxSize, maxAge, maxDocs);
client, maxSize, maxSinglePrimarySize, maxAge, maxDocs);
RolloverStep rolloverStep = new RolloverStep(rolloverStepKey, waitForActiveShardsKey, client);
WaitForActiveShardsStep waitForActiveShardsStep = new WaitForActiveShardsStep(waitForActiveShardsKey, updateDateStepKey);
UpdateRolloverLifecycleDateStep updateDateStep = new UpdateRolloverLifecycleDateStep(updateDateStepKey, setIndexingCompleteStepKey,
Expand All @@ -157,7 +179,7 @@ public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey)

@Override
public int hashCode() {
return Objects.hash(maxSize, maxAge, maxDocs);
return Objects.hash(maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -170,8 +192,9 @@ public boolean equals(Object obj) {
}
RolloverAction other = (RolloverAction) obj;
return Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}

@Override
Expand Down
22 changes: 16 additions & 6 deletions .../plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ public class WaitForRolloverReadyStep extends AsyncWaitStep {
public static final String NAME = "check-rollover-ready";

private final ByteSizeValue maxSize;
private final ByteSizeValue maxSinglePrimarySize;
private final TimeValue maxAge;
private final Long maxDocs;

public WaitForRolloverReadyStep(StepKey key, StepKey nextStepKey, Client client, ByteSizeValue maxSize, TimeValue maxAge,
Long maxDocs) {
public WaitForRolloverReadyStep(StepKey key, StepKey nextStepKey, Client client,
ByteSizeValue maxSize, ByteSizeValue maxSinglePrimarySize, TimeValue maxAge, Long maxDocs) {
super(key, nextStepKey, client);
this.maxSize = maxSize;
this.maxSinglePrimarySize = maxSinglePrimarySize;
this.maxAge = maxAge;
this.maxDocs = maxDocs;
}
Expand Down Expand Up @@ -139,12 +141,15 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener,

RolloverRequest rolloverRequest = new RolloverRequest(rolloverTarget, null).masterNodeTimeout(masterTimeout);
rolloverRequest.dryRun(true);
if (maxAge != null) {
rolloverRequest.addMaxIndexAgeCondition(maxAge);
}
if (maxSize != null) {
rolloverRequest.addMaxIndexSizeCondition(maxSize);
}
if (maxSinglePrimarySize != null) {
rolloverRequest.addMaxSinglePrimarySizeCondition(maxSinglePrimarySize);
}
if (maxAge != null) {
rolloverRequest.addMaxIndexAgeCondition(maxAge);
}
if (maxDocs != null) {
rolloverRequest.addMaxIndexDocsCondition(maxDocs);
}
Expand All @@ -157,6 +162,10 @@ ByteSizeValue getMaxSize() {
return maxSize;
}

ByteSizeValue getMaxSinglePrimarySize() {
return maxSinglePrimarySize;
}

TimeValue getMaxAge() {
return maxAge;
}
Expand All @@ -167,7 +176,7 @@ Long getMaxDocs() {

@Override
public int hashCode() {
return Objects.hash(super.hashCode(), maxSize, maxAge, maxDocs);
return Objects.hash(super.hashCode(), maxSize, maxSinglePrimarySize, maxAge, maxDocs);
}

@Override
Expand All @@ -181,6 +190,7 @@ public boolean equals(Object obj) {
WaitForRolloverReadyStep other = (WaitForRolloverReadyStep) obj;
return super.equals(obj) &&
Objects.equals(maxSize, other.maxSize) &&
Objects.equals(maxSinglePrimarySize, other.maxSinglePrimarySize) &&
Objects.equals(maxAge, other.maxAge) &&
Objects.equals(maxDocs, other.maxDocs);
}
Expand Down

0 comments on commit 128f205

Please sign in to comment.