Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

The ignore_unavailable option should also ignore indices that are closed #6475

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Expand Up @@ -48,7 +48,7 @@ public class MultiPercolateRequest extends ActionRequest<MultiPercolateRequest>

private String[] indices;
private String documentType;
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
private List<PercolateRequest> requests = Lists.newArrayList();

public MultiPercolateRequest add(PercolateRequestBuilder requestBuilder) {
Expand All @@ -62,7 +62,7 @@ public MultiPercolateRequest add(PercolateRequest request) {
if (request.documentType() == null && documentType != null) {
request.documentType(documentType);
}
if (request.indicesOptions() == IndicesOptions.strictExpandOpen() && indicesOptions != IndicesOptions.strictExpandOpen()) {
if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && indicesOptions != IndicesOptions.strictExpandOpenAndForbidClosed()) {
request.indicesOptions(indicesOptions);
}
requests.add(request);
Expand Down Expand Up @@ -96,7 +96,7 @@ public MultiPercolateRequest add(BytesReference data, boolean contentUnsafe, boo
if (documentType != null) {
percolateRequest.documentType(documentType);
}
if (indicesOptions != IndicesOptions.strictExpandOpen()) {
if (indicesOptions != IndicesOptions.strictExpandOpenAndForbidClosed()) {
percolateRequest.indicesOptions(indicesOptions);
}

Expand Down Expand Up @@ -165,10 +165,11 @@ private void parsePercolateAction(XContentParser parser, PercolateRequest percol
}
}

boolean ignoreUnavailable = IndicesOptions.strictExpandOpen().ignoreUnavailable();
boolean allowNoIndices = IndicesOptions.strictExpandOpen().allowNoIndices();
boolean expandWildcardsOpen = IndicesOptions.strictExpandOpen().expandWildcardsOpen();
boolean expandWildcardsClosed = IndicesOptions.strictExpandOpen().expandWildcardsClosed();
IndicesOptions defaultOptions = indicesOptions;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not sure if you wanted to do the following instead?

IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed();

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The indicesOptions field is already set to IndicesOptions.strictExpandOpenAndForbidClosed(), so this should be ok?

boolean ignoreUnavailable = defaultOptions.ignoreUnavailable();
boolean allowNoIndices = defaultOptions.allowNoIndices();
boolean expandWildcardsOpen = defaultOptions.expandWildcardsOpen();
boolean expandWildcardsClosed = defaultOptions.expandWildcardsClosed();

if (header.containsKey("id")) {
GetRequest getRequest = new GetRequest(globalIndex);
Expand Down Expand Up @@ -280,7 +281,7 @@ private void parsePercolateAction(XContentParser parser, PercolateRequest percol
}
}
}
percolateRequest.indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandWildcardsOpen, expandWildcardsClosed));
percolateRequest.indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandWildcardsOpen, expandWildcardsClosed, defaultOptions));
}

private String[] parseArray(XContentParser parser) throws IOException {
Expand Down
Expand Up @@ -48,7 +48,7 @@ public class MultiSearchRequest extends ActionRequest<MultiSearchRequest> {

private List<SearchRequest> requests = Lists.newArrayList();

private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();

/**
* Add a search request to execute. Note, the order is important, the search response will be returned in the
Expand All @@ -70,7 +70,7 @@ public MultiSearchRequest add(SearchRequest request) {

public MultiSearchRequest add(byte[] data, int from, int length, boolean contentUnsafe,
@Nullable String[] indices, @Nullable String[] types, @Nullable String searchType) throws Exception {
return add(new BytesArray(data, from, length), contentUnsafe, indices, types, searchType, null, IndicesOptions.strictExpandOpen(), true);
return add(new BytesArray(data, from, length), contentUnsafe, indices, types, searchType, null, IndicesOptions.strictExpandOpenAndForbidClosed(), true);
}

public MultiSearchRequest add(BytesReference data, boolean contentUnsafe, @Nullable String[] indices, @Nullable String[] types, @Nullable String searchType, IndicesOptions indicesOptions) throws Exception {
Expand Down Expand Up @@ -108,10 +108,11 @@ public MultiSearchRequest add(BytesReference data, boolean contentUnsafe, @Nulla
}
searchRequest.searchType(searchType);

boolean ignoreUnavailable = IndicesOptions.strictExpandOpen().ignoreUnavailable();
boolean allowNoIndices = IndicesOptions.strictExpandOpen().allowNoIndices();
boolean expandWildcardsOpen = IndicesOptions.strictExpandOpen().expandWildcardsOpen();
boolean expandWildcardsClosed = IndicesOptions.strictExpandOpen().expandWildcardsClosed();
IndicesOptions defaultOptions = IndicesOptions.strictExpandOpenAndForbidClosed();
boolean ignoreUnavailable = defaultOptions.ignoreUnavailable();
boolean allowNoIndices = defaultOptions.allowNoIndices();
boolean expandWildcardsOpen = defaultOptions.expandWildcardsOpen();
boolean expandWildcardsClosed = defaultOptions.expandWildcardsClosed();

// now parse the action
if (nextMarker - from > 0) {
Expand Down Expand Up @@ -181,7 +182,7 @@ public MultiSearchRequest add(BytesReference data, boolean contentUnsafe, @Nulla
}
}
}
searchRequest.indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandWildcardsOpen, expandWildcardsClosed));
searchRequest.indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, allowNoIndices, expandWildcardsOpen, expandWildcardsClosed, defaultOptions));

// move pointers
from = nextMarker + 1;
Expand Down
Expand Up @@ -41,7 +41,7 @@ public MultiSearchRequestBuilder(Client client) {
* will not be used (if set).
*/
public MultiSearchRequestBuilder add(SearchRequest request) {
if (request.indicesOptions() == IndicesOptions.strictExpandOpen() && request().indicesOptions() != IndicesOptions.strictExpandOpen()) {
if (request.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
request.indicesOptions(request().indicesOptions());
}

Expand All @@ -54,7 +54,7 @@ public MultiSearchRequestBuilder add(SearchRequest request) {
* same order as the search requests.
*/
public MultiSearchRequestBuilder add(SearchRequestBuilder request) {
if (request.request().indicesOptions() == IndicesOptions.strictExpandOpen() && request().indicesOptions() != IndicesOptions.strictExpandOpen()) {
if (request.request().indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() && request().indicesOptions() != IndicesOptions.strictExpandOpenAndForbidClosed()) {
request.request().indicesOptions(request().indicesOptions());
}

Expand Down
Expand Up @@ -83,7 +83,7 @@ public class SearchRequest extends ActionRequest<SearchRequest> {

private String[] types = Strings.EMPTY_ARRAY;

private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();

public SearchRequest() {
}
Expand Down
51 changes: 41 additions & 10 deletions src/main/java/org/elasticsearch/action/support/IndicesOptions.java
Expand Up @@ -40,9 +40,10 @@ public class IndicesOptions {
private static final byte EXPAND_WILDCARDS_OPEN = 4;
private static final byte EXPAND_WILDCARDS_CLOSED = 8;
private static final byte FORBID_ALIASES_TO_MULTIPLE_INDICES = 16;
private static final byte FORBID_CLOSED_INDICES = 32;

static {
byte max = 1 << 5;
byte max = 1 << 6;
VALUES = new IndicesOptions[max];
for (byte id = 0; id < max; id++) {
VALUES[id] = new IndicesOptions(id);
Expand Down Expand Up @@ -84,6 +85,13 @@ public boolean expandWildcardsClosed() {
return (id & EXPAND_WILDCARDS_CLOSED) != 0;
}

/**
* @return Whether execution on closed indices is allowed.
*/
public boolean forbidClosedIndices() {
return (id & FORBID_CLOSED_INDICES) != 0;
}

/**
* @return whether aliases pointing to multiple indices are allowed
*/
Expand All @@ -94,12 +102,16 @@ public boolean allowAliasesToMultipleIndices() {
}

public void writeIndicesOptions(StreamOutput out) throws IOException {
if (allowAliasesToMultipleIndices() || out.getVersion().onOrAfter(Version.V_1_2_0)) {
if (out.getVersion().onOrAfter(Version.V_1_2_2)) {
out.write(id);
} else {
//if we are talking to a node that doesn't support the newly added flag (allowAliasesToMultipleIndices)
//flip to 0 all the bits starting from the 5th
} else if (out.getVersion().before(Version.V_1_2_0)) {
// Target node doesn't know about the FORBID_CLOSED_INDICES and FORBID_ALIASES_TO_MULTIPLE_INDICES flags,
// so unset the bits starting from the 5th position.
out.write(id & 0xf);
} else {
// Target node doesn't know about the FORBID_CLOSED_INDICES flag,
// so unset the bits starting from the 6th position.
out.write(id & 0x1f);
}
}

Expand All @@ -114,11 +126,15 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
}

public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices) {
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, true);
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, true, false);
}

static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices) {
byte id = toByte(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, allowAliasesToMultipleIndices);
public static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices, IndicesOptions defaultOptions) {
return fromOptions(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, defaultOptions.allowAliasesToMultipleIndices(), defaultOptions.forbidClosedIndices());
}

static IndicesOptions fromOptions(boolean ignoreUnavailable, boolean allowNoIndices, boolean expandToOpenIndices, boolean expandToClosedIndices, boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices) {
byte id = toByte(ignoreUnavailable, allowNoIndices, expandToOpenIndices, expandToClosedIndices, allowAliasesToMultipleIndices, forbidClosedIndices);
return VALUES[id];
}

Expand Down Expand Up @@ -150,7 +166,9 @@ public static IndicesOptions fromRequest(RestRequest request, IndicesOptions def
toBool(sIgnoreUnavailable, defaultSettings.ignoreUnavailable()),
toBool(sAllowNoIndices, defaultSettings.allowNoIndices()),
expandWildcardsOpen,
expandWildcardsClosed
expandWildcardsClosed,
defaultSettings.allowAliasesToMultipleIndices(),
defaultSettings.forbidClosedIndices()
);
}

Expand All @@ -162,6 +180,15 @@ public static IndicesOptions strictExpandOpen() {
return VALUES[6];
}

/**
* @return indices options that requires every specified index to exist, expands wildcards only to open indices,
* allows that no indices are resolved from wildcard expressions (not returning an error) and forbids the
* use of closed indices by throwing an error.
*/
public static IndicesOptions strictExpandOpenAndForbidClosed() {
return VALUES[38];
}

/**
* @return indices option that requires every specified index to exist, expands wildcards to both open and closed
* indices and allows that no indices are resolved from wildcard expressions (not returning an error).
Expand All @@ -186,7 +213,8 @@ public static IndicesOptions lenientExpandOpen() {
return VALUES[7];
}

private static byte toByte(boolean ignoreUnavailable, boolean allowNoIndices, boolean wildcardExpandToOpen, boolean wildcardExpandToClosed, boolean allowAliasesToMultipleIndices) {
private static byte toByte(boolean ignoreUnavailable, boolean allowNoIndices, boolean wildcardExpandToOpen,
boolean wildcardExpandToClosed, boolean allowAliasesToMultipleIndices, boolean forbidClosedIndices) {
byte id = 0;
if (ignoreUnavailable) {
id |= IGNORE_UNAVAILABLE;
Expand All @@ -205,6 +233,9 @@ private static byte toByte(boolean ignoreUnavailable, boolean allowNoIndices, bo
if (!allowAliasesToMultipleIndices) {
id |= FORBID_ALIASES_TO_MULTIPLE_INDICES;
}
if (forbidClosedIndices) {
id |= FORBID_CLOSED_INDICES;
}
return id;
}

Expand Down
Expand Up @@ -34,7 +34,7 @@
public abstract class BroadcastOperationRequest<T extends BroadcastOperationRequest> extends ActionRequest<T> {

protected String[] indices;
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen();
private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed();

protected BroadcastOperationRequest() {

Expand Down
45 changes: 37 additions & 8 deletions src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
Expand Up @@ -41,6 +41,7 @@
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.index.Index;
import org.elasticsearch.indices.IndexClosedException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.warmer.IndexWarmersMetaData;
Expand Down Expand Up @@ -623,8 +624,10 @@ private Map<String, Set<String>> resolveSearchRoutingAllIndices(String routing)
return null;
}


/**
* Translates the provided indices or aliases, eventually containing wildcard expressions, into actual indices.
*
* @param indicesOptions how the aliases or indices need to be resolved to concrete indices
* @param aliasesOrIndices the aliases or indices to be resolved to concrete indices
* @return the obtained concrete indices
Expand All @@ -635,7 +638,6 @@ private Map<String, Set<String>> resolveSearchRoutingAllIndices(String routing)
* indices options don't allow such a case.
*/
public String[] concreteIndices(IndicesOptions indicesOptions, String... aliasesOrIndices) throws IndexMissingException, ElasticsearchIllegalArgumentException {

if (indicesOptions.expandWildcardsOpen() || indicesOptions.expandWildcardsClosed()) {
if (isAllIndices(aliasesOrIndices)) {
String[] concreteIndices;
Expand All @@ -655,27 +657,44 @@ public String[] concreteIndices(IndicesOptions indicesOptions, String... aliases

aliasesOrIndices = convertFromWildcards(aliasesOrIndices, indicesOptions);
}
boolean failClosed = indicesOptions.forbidClosedIndices() && !indicesOptions.ignoreUnavailable();

// optimize for single element index (common case)
if (aliasesOrIndices.length == 1) {
return concreteIndices(aliasesOrIndices[0], indicesOptions.allowNoIndices(), indicesOptions.allowAliasesToMultipleIndices());
return concreteIndices(aliasesOrIndices[0], indicesOptions.allowNoIndices(), failClosed, indicesOptions.allowAliasesToMultipleIndices());
}

// check if its a possible aliased index, if not, just return the passed array
boolean possiblyAliased = false;
boolean closedIndices = false;
for (String index : aliasesOrIndices) {
if (!this.indices.containsKey(index)) {
IndexMetaData indexMetaData = indices.get(index);
if (indexMetaData == null) {
possiblyAliased = true;
break;
} else {
if (indicesOptions.forbidClosedIndices() && indexMetaData.getState() == IndexMetaData.State.CLOSE) {
if (failClosed) {
throw new IndexClosedException(new Index(index));
} else {
closedIndices = true;
}
}
}
}
if (!possiblyAliased) {
return aliasesOrIndices;
if (closedIndices) {
Set<String> actualIndices = new HashSet<>(Arrays.asList(aliasesOrIndices));
actualIndices.retainAll(new HashSet<Object>(Arrays.asList(allOpenIndices)));
return actualIndices.toArray(new String[actualIndices.size()]);
} else {
return aliasesOrIndices;
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This code block is an optimization that's useful in case there's no aliases in the input argument and we can return that same array quickly as we received it. I wonder if it still makes sense to have this optimization given that we might need to modify the array. I would love to remove it as it adds complexity... What do you think @martijnvg?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most of the times requested indices aren't closed indices, so I think in that case the additional checks is worth it.

}

Set<String> actualIndices = new HashSet<>();
for (String aliasOrIndex : aliasesOrIndices) {
String[] indices = concreteIndices(aliasOrIndex, indicesOptions.ignoreUnavailable(), indicesOptions.allowAliasesToMultipleIndices());
String[] indices = concreteIndices(aliasOrIndex, indicesOptions.ignoreUnavailable(), failClosed, indicesOptions.allowAliasesToMultipleIndices());
Collections.addAll(actualIndices, indices);
}

Expand All @@ -691,10 +710,15 @@ public String concreteSingleIndex(String indexOrAlias) throws IndexMissingExcept
return indices[0];
}

private String[] concreteIndices(String aliasOrIndex, boolean allowNoIndices, boolean allowMultipleIndices) throws IndexMissingException, ElasticsearchIllegalArgumentException {
private String[] concreteIndices(String aliasOrIndex, boolean allowNoIndices, boolean failClosed, boolean allowMultipleIndices) throws IndexMissingException, ElasticsearchIllegalArgumentException {
// a quick check, if this is an actual index, if so, return it
if (indices.containsKey(aliasOrIndex)) {
return new String[]{aliasOrIndex};
IndexMetaData indexMetaData = indices.get(aliasOrIndex);
if (indexMetaData != null) {
if (indexMetaData.getState() == IndexMetaData.State.CLOSE && failClosed) {
throw new IndexClosedException(new Index(aliasOrIndex));
} else {
return new String[]{aliasOrIndex};
}
}
// not an actual index, fetch from an alias
String[] indices = aliasAndIndexToIndexMap.getOrDefault(aliasOrIndex, Strings.EMPTY_ARRAY);
Expand All @@ -704,6 +728,11 @@ private String[] concreteIndices(String aliasOrIndex, boolean allowNoIndices, bo
if (indices.length > 1 && !allowMultipleIndices) {
throw new ElasticsearchIllegalArgumentException("Alias [" + aliasOrIndex + "] has more than one indices associated with it [" + Arrays.toString(indices) + "], can't execute a single index op");
}

indexMetaData = this.indices.get(aliasOrIndex);
if (indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE && failClosed) {
throw new IndexClosedException(new Index(aliasOrIndex));
}
return indices;
}

Expand Down