Skip to content

Commit

Permalink
Fix date math hidden index resolution (#65278)
Browse files Browse the repository at this point in the history 
This commit updates the IndexAbstractionResolver so that hidden indices
are properly resolved when date math is in use and when we are checking
if the index is visible.

Closes #65157
Backport of #65236
  • Loading branch information
@jaymode
jaymode committed Nov 19, 2020
1 parent b60d071 commit 893e1a5
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 20 deletions.
13 changes: 7 additions & 6 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,16 +146,17 @@ public static boolean isIndexVisible(String expression, String index, IndicesOpt
return includeDataStreams;
}
assert indexAbstraction.getIndices().size() == 1 : "concrete index must point to a single index";
IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
return false;
}

// the index is not hidden and since it is a date math expression, we consider it visible regardless of open/closed
// since it is a date math expression, we consider the index visible regardless of open/closed/hidden as the user is using
// date math to explicitly reference the index
if (dateMathExpression) {
assert IndexMetadata.State.values().length == 2 : "a new IndexMetadata.State value may need to be handled!";
return true;
}
if (isHidden && indicesOptions.expandWildcardsHidden() == false && isVisibleDueToImplicitHidden(expression, index) == false) {
return false;
}

IndexMetadata indexMetadata = indexAbstraction.getIndices().get(0);
if (indexMetadata.getState() == IndexMetadata.State.CLOSE && indicesOptions.expandWildcardsClosed()) {
return true;
}
Expand Down
29 changes: 28 additions & 1 deletion server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,13 @@
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.ESTestCase;

import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand All @@ -46,14 +50,17 @@
import java.util.Set;
import java.util.stream.Collectors;

import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.elasticsearch.cluster.DataStreamTestHelper.createTimestampField;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.oneOf;
import static org.hamcrest.core.IsNull.notNullValue;

public class ResolveIndexTests extends ESTestCase {

private final Object[][] indices = new Object[][]{
private final Object[][] indices = new Object[][] {
// name, isClosed, isHidden, isFrozen, dataStream, aliases
{"logs-pgsql-prod-20200101", false, false, true, null, new String[]{"logs-pgsql-prod"}},
{"logs-pgsql-prod-20200102", false, false, true, null, new String[]{"logs-pgsql-prod", "one-off-alias"}},
Expand Down Expand Up @@ -169,6 +176,26 @@ public void testResolveWithMultipleNames() {
validateDataStreams(dataStreams, "logs-mysql-test");
}

public void testResolveHiddenProperlyWithDateMath() {
// set up with today's index and following day's index to avoid test failures due to execution time
DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
Instant now = Instant.now(Clock.systemUTC());
String todaySuffix = dateFormatter.format(now);
String tomorrowSuffix = dateFormatter.format(now.plus(Duration.ofDays(1L)));
Object[][] indices = new Object[][] {
// name, isClosed, isHidden, isFrozen, dataStream, aliases
{"logs-pgsql-prod-" + todaySuffix, false, true, false, null, Strings.EMPTY_ARRAY},
{"logs-pgsql-prod-" + tomorrowSuffix, false, true, false, null, Strings.EMPTY_ARRAY}
};
Metadata metadata = buildMetadata(new Object[][] {}, indices);

String requestedIndex = "<logs-pgsql-prod-{now/d}>";
List<String> resolvedIndices = resolver.resolveIndexAbstractions(singletonList(requestedIndex), IndicesOptions.LENIENT_EXPAND_OPEN,
metadata, asList("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix), randomBoolean(), randomBoolean());
assertThat(resolvedIndices.size(), is(1));
assertThat(resolvedIndices.get(0), oneOf("logs-pgsql-prod-" + todaySuffix, "logs-pgsql-prod-" + tomorrowSuffix));
}

private void validateIndices(List<ResolvedIndex> resolvedIndices, String... expectedIndices) {
assertThat(resolvedIndices.size(), equalTo(expectedIndices.length));
for (int k = 0; k < resolvedIndices.size(); k++) {
Expand Down
55 changes: 42 additions & 13 deletions .../src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
Expand Down Expand Up @@ -74,6 +75,9 @@
import org.joda.time.format.DateTimeFormat;
import org.junit.Before;

import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
Expand All @@ -97,6 +101,7 @@
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.oneOf;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doCallRealMethod;
Expand All @@ -113,6 +118,8 @@ public class IndicesAndAliasesResolverTests extends ESTestCase {
private IndicesAndAliasesResolver defaultIndicesResolver;
private IndexNameExpressionResolver indexNameExpressionResolver;
private Map<String, RoleDescriptor> roleMap;
private String todaySuffix;
private String tomorrowSuffix;

@Before
public void setup() {
Expand All @@ -126,6 +133,10 @@ public void setup() {

indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));

DateFormatter dateFormatter = DateFormatter.forPattern("uuuu.MM.dd");
Instant now = Instant.now(Clock.systemUTC());
todaySuffix = dateFormatter.format(now);
tomorrowSuffix = dateFormatter.format(now.plus(Duration.ofDays(1L)));
final boolean withAlias = randomBoolean();
final String securityIndexName = SECURITY_MAIN_ALIAS + (withAlias ? "-" + randomAlphaOfLength(5) : "");
final String dataStreamName = "logs-foobar";
Expand Down Expand Up @@ -169,6 +180,10 @@ public void setup() {
.put(indexBuilder("visible-w-aliases").settings(Settings.builder().put(settings).build())
.putAlias(AliasMetadata.builder("alias-visible").build())
.putAlias(AliasMetadata.builder("alias-visible-mixed").isHidden(false).build()))
.put(indexBuilder("date-hidden-" + todaySuffix)
.settings(Settings.builder().put(settings).put("index.hidden", true).build()))
.put(indexBuilder("date-hidden-" + tomorrowSuffix)
.settings(Settings.builder().put(settings).put("index.hidden", true).build()))
.put(dataStreamIndex1, true)
.put(dataStreamIndex2, true)
.put(dataStreamIndex3, true)
Expand All @@ -188,7 +203,8 @@ public void setup() {
userNoIndices = new User("test", "test");
rolesStore = mock(CompositeRolesStore.class);
String[] authorizedIndices = new String[] { "bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "missing", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
String[] dashIndices = new String[]{"-index10", "-index11", "-index20", "-index21"};
roleMap = new HashMap<>();
roleMap.put("role", new RoleDescriptor("role", null,
Expand Down Expand Up @@ -1042,10 +1058,11 @@ public void testResolveAllGetAliasesRequest() {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "alias1",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
String[] replacedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(replacedIndices));
assertThat(request.aliases(), arrayContainingInAnyOrder("alias1"));
Expand Down Expand Up @@ -1123,7 +1140,7 @@ public void testResolveAllAliasesGetAliasesRequest() {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(expectedIndices));
Expand All @@ -1139,12 +1156,14 @@ public void testResolveAllAndExplicitAliasesGetAliasesRequest() {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "explicit",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix};
logger.info("indices: {}", indices);
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(request.aliases(), arrayContainingInAnyOrder("foofoobar", "foobarfoo", "explicit"));
}

Expand All @@ -1157,7 +1176,7 @@ public void testResolveAllAndWildcardsAliasesGetAliasesRequest() {
List<String> indices = resolveIndices(request, authorizedIndices).getLocal();
//the union of all resolved indices and aliases gets returned, including hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//_all gets replaced with all indices that user is authorized for
assertThat(request.indices(), arrayContainingInAnyOrder(expectedIndices));
Expand Down Expand Up @@ -1199,11 +1218,12 @@ public void testResolveAliasesExclusionWildcardsGetAliasesRequest() {
//the same and resolve those aliases to their corresponding concrete indices (which we let core do)
//also includes hidden indices as Get Aliases includes hidden by default
String[] expectedIndices = new String[]{"bar", "bar-closed", "foobarfoo", "foofoo", "foofoo-closed", "foofoobar", "hidden-open",
"hidden-closed", ".hidden-open", ".hidden-closed"};
"hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix};
assertSameValues(indices, expectedIndices);
//alias foofoobar on both sides, that's fine, es core would do the same, same as above
assertThat(request.indices(), arrayContainingInAnyOrder("bar", "bar-closed", "foobarfoo", "foofoo", "foofoo-closed", "foofoobar",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(request.aliases(), arrayContainingInAnyOrder("foofoobar"));
}

Expand Down Expand Up @@ -1486,19 +1506,21 @@ public void testHiddenIndicesResolution() {
List<String> authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
ResolvedIndices resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), containsInAnyOrder("bar", "bar-closed", "foofoobar", "foobarfoo", "foofoo", "foofoo-closed",
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed"));
"hidden-open", "hidden-closed", ".hidden-open", ".hidden-closed", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(resolvedIndices.getRemote(), emptyIterable());

// open + hidden
searchRequest = new SearchRequest();
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, true));
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(),
containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open"));
containsInAnyOrder("bar", "foofoobar", "foobarfoo", "foofoo", "hidden-open", ".hidden-open", "date-hidden-" + todaySuffix,
"date-hidden-" + tomorrowSuffix));
assertThat(resolvedIndices.getRemote(), emptyIterable());

// open + implicit hidden for . indices
searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
searchRequest = new SearchRequest(randomFrom(".h*", ".hid*"));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, true, false, false));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
Expand All @@ -1514,7 +1536,7 @@ public void testHiddenIndicesResolution() {
assertThat(resolvedIndices.getRemote(), emptyIterable());

// closed + implicit hidden for . indices
searchRequest = new SearchRequest(randomFrom(".*", ".hid*"));
searchRequest = new SearchRequest(randomFrom(".h*", ".hid*"));
searchRequest.indicesOptions(IndicesOptions.fromOptions(false, false, false, true, false));
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
Expand All @@ -1528,6 +1550,13 @@ public void testHiddenIndicesResolution() {
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains("-*"));
assertThat(resolvedIndices.getRemote(), emptyIterable());

// date math with default indices options
searchRequest = new SearchRequest("<date-hidden-{now/d}>");
authorizedIndices = buildAuthorizedIndices(user, SearchAction.NAME);
resolvedIndices = defaultIndicesResolver.resolveIndicesAndAliases(searchRequest, metadata, authorizedIndices);
assertThat(resolvedIndices.getLocal(), contains(oneOf("date-hidden-" + todaySuffix, "date-hidden-" + tomorrowSuffix)));
assertThat(resolvedIndices.getRemote(), emptyIterable());
}

public void testHiddenAliasesResolution() {
Expand Down

0 comments on commit 893e1a5

Please sign in to comment.