Skip to content

Commit

Permalink
Refactor WildcardExpressionResolver for reusability (#89311)
Browse files Browse the repository at this point in the history 
This is a pure refactoring that shouldn't exhibit any behavior change.
The goal here is to specify the interface of
`WildcardExpressionResolver#innerResolve`, i.e. what are the properties
of its return value. The ultimate goal is to facilitate the replacement
of `IndicesAndAliasesResolver#resolveIndicesAndAliases`.
  • Loading branch information
@albertzaharovits
albertzaharovits committed Aug 23, 2022
1 parent 327b8f8 commit 2b6fdfd
Show file tree
Hide file tree
Showing 3 changed files with 198 additions and 234 deletions.
197 changes: 108 additions & 89 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.function.Consumer;
import java.util.function.LongSupplier;
import java.util.function.Predicate;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -188,8 +189,8 @@ public List<String> dataStreamNames(ClusterState state, IndicesOptions options,
indexExpressions = new String[] { "*" };
}

final List<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);
return ((expressions == null) ? List.<String>of() : expressions).stream()
final Collection<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);
return expressions.stream()
.map(x -> state.metadata().getIndicesLookup().get(x))
.filter(Objects::nonNull)
.filter(ia -> ia.getType() == IndexAbstraction.Type.DATA_STREAM)
Expand All @@ -213,18 +214,17 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit
false,
false,
includeDataStreams,
true,
getSystemIndexAccessLevel(),
getSystemIndexAccessPredicate(),
getNetNewSystemIndexPredicate()
);

final List<String> expressions = resolveExpressions(List.of(request.index()), context);
final Collection<String> expressions = resolveExpressions(List.of(request.index()), context);

if (expressions.size() == 1) {
IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.get(0));
IndexAbstraction ia = state.metadata().getIndicesLookup().get(expressions.iterator().next());
if (ia == null) {
throw new IndexNotFoundException(expressions.get(0));
throw new IndexNotFoundException(expressions.iterator().next());
}
if (ia.getType() == IndexAbstraction.Type.ALIAS) {
Index writeIndex = ia.getWriteIndex();
Expand All @@ -247,7 +247,7 @@ public IndexAbstraction resolveWriteIndexAbstraction(ClusterState state, DocWrit
}
}

private static List<String> resolveExpressions(List<String> expressions, Context context) {
private static Collection<String> resolveExpressions(List<String> expressions, Context context) {
return WildcardExpressionResolver.resolve(context, DateMathExpressionResolver.resolve(context, expressions));
}

Expand Down Expand Up @@ -341,7 +341,7 @@ Index[] concreteIndices(Context context, String... indexExpressions) {
final boolean failNoIndices = indexExpressions.length == 1
? options.allowNoIndices() == false
: options.ignoreUnavailable() == false;
final List<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);
final Collection<String> expressions = resolveExpressions(Arrays.asList(indexExpressions), context);

if (expressions.isEmpty()) {
if (options.allowNoIndices() == false) {
Expand Down Expand Up @@ -664,7 +664,13 @@ public Set<String> resolveExpressions(ClusterState state, String... expressions)
getSystemIndexAccessPredicate(),
getNetNewSystemIndexPredicate()
);
return Set.copyOf(resolveExpressions(Arrays.asList(expressions), context));
Collection<String> resolved = resolveExpressions(Arrays.asList(expressions), context);
if (resolved instanceof Set<String>) {
// unmodifiable without creating a new collection as it might contain many items
return Collections.unmodifiableSet((Set<String>) resolved);
} else {
return Set.copyOf(resolved);
}
}

/**
Expand Down Expand Up @@ -785,7 +791,7 @@ public Map<String, Set<String>> resolveSearchRouting(ClusterState state, @Nullab
getSystemIndexAccessPredicate(),
getNetNewSystemIndexPredicate()
);
final List<String> resolvedExpressions = resolveExpressions(
final Collection<String> resolvedExpressions = resolveExpressions(
expressions != null ? Arrays.asList(expressions) : Collections.emptyList(),
context
);
Expand Down Expand Up @@ -1124,48 +1130,60 @@ private WildcardExpressionResolver() {
// Utility class
}

public static List<String> resolve(Context context, List<String> expressions) {
/**
* Returns a collection of resource names given the {@param expressions} which contains wildcards and exclusions.
*/
public static Collection<String> resolve(Context context, List<String> expressions) {
Objects.requireNonNull(expressions);
// only check open/closed since if we do not expand to open or closed it doesn't make sense to
// expand to hidden
if (context.getOptions().expandWildcardsClosed() == false && context.getOptions().expandWildcardsOpen() == false) {
return expressions;
} else if (isEmptyOrTrivialWildcard(expressions)) {
return innerResolveAll(context);
} else {
return innerResolve(context, expressions);
}
}

if (isEmptyOrTrivialWildcard(expressions)) {
List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(context);
if (context.includeDataStreams()) {
final Map<String, IndexAbstraction> dataStreamsAbstractions = context.getState()
.metadata()
.getIndicesLookup()
.entrySet()
.stream()
.filter(entry -> entry.getValue().getType() == IndexAbstraction.Type.DATA_STREAM)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
// dedup backing indices if expand hidden indices option is true
Set<String> resolvedIncludingDataStreams = new HashSet<>(resolvedExpressions);
resolvedIncludingDataStreams.addAll(
expand(context, dataStreamsAbstractions, expressions.isEmpty() ? "_all" : expressions.get(0))
);
return new ArrayList<>(resolvedIncludingDataStreams);
}
/**
* Returns all the indices and all the datastreams, considering the open/closed, system, and hidden context parameters.
* Depending on the context, returns the names of the datastreams themselves or their backing indices.
*/
private static Collection<String> innerResolveAll(Context context) {
List<String> resolvedExpressions = resolveEmptyOrTrivialWildcard(context);
if (context.includeDataStreams() == false) {
return resolvedExpressions;
} else {
final List<IndexAbstraction> dataStreamsAbstractions = context.getState()
.metadata()
.getIndicesLookup()
.values()
.stream()
.filter(indexAbstraction -> indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM)
.collect(Collectors.toList());
// dedup backing indices if expand hidden indices option is true
Set<String> resolvedIncludingDataStreams = new HashSet<>(resolvedExpressions);
expandMatches(context, dataStreamsAbstractions, false, resolvedIncludingDataStreams::add);
return resolvedIncludingDataStreams;
}

Set<String> result = innerResolve(context, expressions);

if (result == null) {
return expressions;
}
if (result.isEmpty() && context.getOptions().allowNoIndices() == false) {
IndexNotFoundException infe = new IndexNotFoundException((String) null);
infe.setResources("index_or_alias", expressions.toArray(new String[0]));
throw infe;
}
return new ArrayList<>(result);
}

private static Set<String> innerResolve(Context context, List<String> expressions) {
Set<String> result = null;
/**
* Returns all the existing resource (index, alias and datastream) names that the {@param expressions} resolve to.
* The passed-in {@param expressions} might contain wildcards and exclusions, as well as plain resource names.
* The return is a {@code Collection} (usually a {@code Set} but can also be a {@code List}, for performance reasons) of plain
* resource names only. All the returned resources are "accessible", in the given context, i.e. the resources exist
* and are not an alias or a datastream if the context does not permit it.
* Wildcard expressions, depending on the context:
* - might throw an exception if they don't resolve to anything
* - might not resolve to hidden or system resources (but plain names can refer to hidden or system resources)
* - might resolve to aliases and datastreams, and it could be (depending on the context) that their backing indices are what's
* ultimately returned, instead of the alias or datastream name
*/
private static Collection<String> innerResolve(Context context, List<String> expressions) {
Objects.requireNonNull(expressions);
Collection<String> result = null;
boolean wildcardSeen = false;
for (int i = 0; i < expressions.size(); i++) {
String expression = expressions.get(i);
Expand All @@ -1176,17 +1194,17 @@ private static Set<String> innerResolve(Context context, List<String> expression
}
continue;
}
if (result == null) {
// add all the previous ones...
result = new HashSet<>(expressions.subList(0, i));
}
final boolean add;
if (expression.charAt(0) == '-' && wildcardSeen) {
add = false;
expression = expression.substring(1);
} else {
add = true;
}
if (result == null) {
// add all the previous ones...
result = new HashSet<>(expressions.subList(0, i));
}
if (Regex.isSimpleMatchPattern(expression) == false) {
// TODO why does wildcard resolver throw exceptions regarding non wildcarded expressions? This should not be done here.
if (context.getOptions().ignoreUnavailable() == false) {
Expand All @@ -1200,17 +1218,26 @@ private static Set<String> innerResolve(Context context, List<String> expression
continue;
}
wildcardSeen = true;

final Map<String, IndexAbstraction> matches = matches(context, expression);
if (context.getOptions().allowNoIndices() == false && matches.isEmpty()) {
throw indexNotFoundException(expression);
}
Set<String> expand = expand(context, matches, expression);
if (add) {
result.addAll(expand);
} else {
result.removeAll(expand);
}
Collection<String> finalResult = result;
expandMatches(context, matches.values(), expression.startsWith("."), expanded -> {
if (add) {
finalResult.add(expanded);
} else {
finalResult.remove(expanded);
}
});
}
if (result == null) {
result = expressions;
}
if (result.isEmpty() && context.getOptions().allowNoIndices() == false) {
IndexNotFoundException infe = new IndexNotFoundException((String) null);
infe.setResources("index_or_alias", expressions.toArray(new String[0]));
throw infe;
}
return result;
}
Expand Down Expand Up @@ -1277,7 +1304,7 @@ private static IndexMetadata.State excludeState(IndicesOptions options) {
return excludeState;
}

public static Map<String, IndexAbstraction> matches(Context context, String expression) {
private static Map<String, IndexAbstraction> matches(Context context, String expression) {
SortedMap<String, IndexAbstraction> indicesLookup = context.getState().getMetadata().getIndicesLookup();
if (Regex.isMatchAllPattern(expression)) {
return filterIndicesLookup(indicesLookup, null, context.getOptions().ignoreAliases(), context.includeDataStreams());
Expand Down Expand Up @@ -1335,47 +1362,39 @@ private static Map<String, IndexAbstraction> filterIndicesLookup(
}
}

private static Set<String> expand(Context context, Map<String, IndexAbstraction> matches, String expression) {
private static void expandMatches(
Context context,
Collection<IndexAbstraction> matches,
final boolean expressionStartsWithDot,
Consumer<String> expandConsumer
) {
final IndexMetadata.State excludeState = excludeState(context.getOptions());
final boolean includeHidden = context.getOptions().expandWildcardsHidden();
Set<String> expand = new HashSet<>();
for (Map.Entry<String, IndexAbstraction> entry : matches.entrySet()) {
String aliasOrIndexName = entry.getKey();
IndexAbstraction indexAbstraction = entry.getValue();

if (indexAbstraction.isSystem()) {
if (context.netNewSystemIndexPredicate.test(indexAbstraction.getName())
&& context.systemIndexAccessPredicate.test(indexAbstraction.getName()) == false) {
continue;
}
if (indexAbstraction.getType() == Type.DATA_STREAM || indexAbstraction.getParentDataStream() != null) {
if (context.systemIndexAccessPredicate.test(indexAbstraction.getName()) == false) {
continue;
}
}
for (IndexAbstraction indexAbstraction : matches) {
if (indexAbstraction.isSystem()
&& (indexAbstraction.getType() == Type.DATA_STREAM
|| indexAbstraction.getParentDataStream() != null
|| context.netNewSystemIndexPredicate.test(indexAbstraction.getName()))
&& context.systemIndexAccessPredicate.test(indexAbstraction.getName()) == false) {
continue;
}

if (indexAbstraction.isHidden() == false || includeHidden || implicitHiddenMatch(aliasOrIndexName, expression)) {
if (context.isPreserveAliases() && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
expand.add(aliasOrIndexName);
} else {
for (Index index : indexAbstraction.getIndices()) {
IndexMetadata meta = context.state.metadata().index(index);
if (excludeState == null || meta.getState() != excludeState) {
expand.add(meta.getIndex().getName());
}
}
if (context.isPreserveDataStreams() && indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) {
expand.add(indexAbstraction.getName());
if (indexAbstraction.isHidden()
&& context.getOptions().expandWildcardsHidden() == false
&& (indexAbstraction.getName().startsWith(".") && expressionStartsWithDot) == false) {
continue;
}
if (context.isPreserveAliases() && indexAbstraction.getType() == IndexAbstraction.Type.ALIAS) {
expandConsumer.accept(indexAbstraction.getName());
} else if (context.isPreserveDataStreams() && indexAbstraction.getType() == Type.DATA_STREAM) {
expandConsumer.accept(indexAbstraction.getName());
} else {
for (Index index : indexAbstraction.getIndices()) {
IndexMetadata meta = context.state.metadata().index(index);
if (excludeState == null || meta.getState() != excludeState) {
expandConsumer.accept(meta.getIndex().getName());
}
}
}
}
return expand;
}

private static boolean implicitHiddenMatch(String itemName, String expression) {
return itemName.startsWith(".") && expression.startsWith(".") && Regex.isSimpleMatchPattern(expression);
}

private static boolean isEmptyOrTrivialWildcard(List<String> expressions) {
Expand Down

0 comments on commit 2b6fdfd

Please sign in to comment.