From facec187bbee24e3c3c43547e5d785191f2fd6d5 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 7 Sep 2018 18:16:07 -0700 Subject: [PATCH 01/12] Painless: Add Imported Static Method (#33440) Allow static methods to be imported in Painless and called using just the method name. --- .../elasticsearch/painless/spi/Whitelist.java | 11 +- .../painless/spi/WhitelistLoader.java | 51 +++--- .../elasticsearch/painless/FeatureTest.java | 26 ++- .../painless/lookup/PainlessLookup.java | 15 ++ .../lookup/PainlessLookupBuilder.java | 171 +++++++++++++++++- .../painless/node/ECallLocal.java | 51 ++++-- .../painless/spi/org.elasticsearch.txt | 3 +- .../elasticsearch/painless/BasicAPITests.java | 4 + .../painless/ScriptTestCase.java | 8 +- 9 files changed, 282 insertions(+), 58 deletions(-) diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java index 7acbff6cb0b93..31a9e595d0b6d 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java @@ -61,12 +61,19 @@ public final class Whitelist { /** The {@link List} of all the whitelisted Painless classes. */ public final List whitelistClasses; + /** The {@link List} of all the whitelisted static Painless methods. */ + public final List whitelistImportedMethods; + + /** The {@link List} of all the whitelisted Painless bindings. */ public final List whitelistBindings; - /** Standard constructor. All values must be not {@code null}. */ - public Whitelist(ClassLoader classLoader, List whitelistClasses, List whitelistBindings) { + /** Standard constructor. All values must be not {@code null}. */ + public Whitelist(ClassLoader classLoader, List whitelistClasses, + List whitelistImportedMethods, List whitelistBindings) { + this.classLoader = Objects.requireNonNull(classLoader); this.whitelistClasses = Collections.unmodifiableList(Objects.requireNonNull(whitelistClasses)); + this.whitelistImportedMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistImportedMethods)); this.whitelistBindings = Collections.unmodifiableList(Objects.requireNonNull(whitelistBindings)); } } diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java index 0279c82f1b67b..2f5dec769fc2f 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java @@ -133,6 +133,7 @@ public final class WhitelistLoader { */ public static Whitelist loadFromResourceFiles(Class resource, String... filepaths) { List whitelistClasses = new ArrayList<>(); + List whitelistStatics = new ArrayList<>(); List whitelistBindings = new ArrayList<>(); // Execute a single pass through the whitelist text files. This will gather all the @@ -192,18 +193,18 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep whitelistConstructors = new ArrayList<>(); whitelistMethods = new ArrayList<>(); whitelistFields = new ArrayList<>(); - } else if (line.startsWith("static ")) { + } else if (line.startsWith("static_import ")) { // Ensure the final token of the line is '{'. if (line.endsWith("{") == false) { throw new IllegalArgumentException( - "invalid static definition: failed to parse static opening bracket [" + line + "]"); + "invalid static import definition: failed to parse static import opening bracket [" + line + "]"); } if (parseType != null) { - throw new IllegalArgumentException("invalid definition: cannot embed static definition [" + line + "]"); + throw new IllegalArgumentException("invalid definition: cannot embed static import definition [" + line + "]"); } - parseType = "static"; + parseType = "static_import"; // Handle the end of a definition and reset all previously gathered values. // Expects the following format: '}' '\n' @@ -229,9 +230,9 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep // Reset the parseType. parseType = null; - // Handle static definition types. - // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' 'bound_to' ID '\n' - } else if ("static".equals(parseType)) { + // Handle static import definition types. + // Expects the following format: ID ID '(' ( ID ( ',' ID )* )? ')' ( 'from_class' | 'bound_to' ) ID '\n' + } else if ("static_import".equals(parseType)) { // Mark the origin of this parsable object. String origin = "[" + filepath + "]:[" + number + "]"; @@ -240,7 +241,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep if (parameterStartIndex == -1) { throw new IllegalArgumentException( - "illegal static definition: start of method parameters not found [" + line + "]"); + "illegal static import definition: start of method parameters not found [" + line + "]"); } String[] tokens = line.substring(0, parameterStartIndex).trim().split("\\s+"); @@ -261,7 +262,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep if (parameterEndIndex == -1) { throw new IllegalArgumentException( - "illegal static definition: end of method parameters not found [" + line + "]"); + "illegal static import definition: end of method parameters not found [" + line + "]"); } String[] canonicalTypeNameParameters = @@ -272,39 +273,37 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep canonicalTypeNameParameters = new String[0]; } - // Parse the static type and class. + // Parse the static import type and class. tokens = line.substring(parameterEndIndex + 1).trim().split("\\s+"); - String staticType; + String staticImportType; String targetJavaClassName; // Based on the number of tokens, look up the type and class. if (tokens.length == 2) { - staticType = tokens[0]; + staticImportType = tokens[0]; targetJavaClassName = tokens[1]; } else { - throw new IllegalArgumentException("invalid static definition: unexpected format [" + line + "]"); + throw new IllegalArgumentException("invalid static import definition: unexpected format [" + line + "]"); } - // Check the static type is valid. - if ("bound_to".equals(staticType) == false) { - throw new IllegalArgumentException( - "invalid static definition: unexpected static type [" + staticType + "] [" + line + "]"); + // Add a static import method or binding depending on the static import type. + if ("from_class".equals(staticImportType)) { + whitelistStatics.add(new WhitelistMethod(origin, targetJavaClassName, + methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); + } else if ("bound_to".equals(staticImportType)) { + whitelistBindings.add(new WhitelistBinding(origin, targetJavaClassName, + methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); + } else { + throw new IllegalArgumentException("invalid static import definition: " + + "unexpected static import type [" + staticImportType + "] [" + line + "]"); } - whitelistBindings.add(new WhitelistBinding(origin, targetJavaClassName, - methodName, returnCanonicalTypeName, Arrays.asList(canonicalTypeNameParameters))); - // Handle class definition types. } else if ("class".equals(parseType)) { // Mark the origin of this parsable object. String origin = "[" + filepath + "]:[" + number + "]"; - // Ensure we have a defined class before adding any constructors, methods, augmented methods, or fields. - if (parseType == null) { - throw new IllegalArgumentException("invalid definition: expected one of ['class', 'static'] [" + line + "]"); - } - // Handle the case for a constructor definition. // Expects the following format: '(' ( ID ( ',' ID )* )? ')' '\n' if (line.startsWith("(")) { @@ -393,7 +392,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep ClassLoader loader = AccessController.doPrivileged((PrivilegedAction)resource::getClassLoader); - return new Whitelist(loader, whitelistClasses, whitelistBindings); + return new Whitelist(loader, whitelistClasses, whitelistStatics, whitelistBindings); } private WhitelistLoader() {} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java index 1e94c19f6d90e..28cbb4aee19a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FeatureTest.java @@ -24,6 +24,21 @@ /** Currently just a dummy class for testing a few features not yet exposed by whitelist! */ public class FeatureTest { + /** static method that returns true */ + public static boolean overloadedStatic() { + return true; + } + + /** static method that returns what you ask it */ + public static boolean overloadedStatic(boolean whatToReturn) { + return whatToReturn; + } + + /** static method only whitelisted as a static */ + public static float staticAddFloatsTest(float x, float y) { + return x + y; + } + private int x; private int y; public int z; @@ -58,21 +73,12 @@ public void setY(int y) { this.y = y; } - /** static method that returns true */ - public static boolean overloadedStatic() { - return true; - } - - /** static method that returns what you ask it */ - public static boolean overloadedStatic(boolean whatToReturn) { - return whatToReturn; - } - /** method taking two functions! */ public Object twoFunctionsOfX(Function f, Function g) { return f.apply(g.apply(x)); } + /** method to take in a list */ public void listInput(List list) { } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 2d6ed3e361dc3..7be659d11a124 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -37,16 +37,23 @@ public final class PainlessLookup { private final Map> canonicalClassNamesToClasses; private final Map, PainlessClass> classesToPainlessClasses; + private final Map painlessMethodKeysToImportedPainlessMethods; private final Map painlessMethodKeysToPainlessBindings; PainlessLookup(Map> canonicalClassNamesToClasses, Map, PainlessClass> classesToPainlessClasses, + Map painlessMethodKeysToImportedPainlessMethods, Map painlessMethodKeysToPainlessBindings) { + Objects.requireNonNull(canonicalClassNamesToClasses); Objects.requireNonNull(classesToPainlessClasses); + Objects.requireNonNull(painlessMethodKeysToImportedPainlessMethods); + Objects.requireNonNull(painlessMethodKeysToPainlessBindings); + this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses); this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses); + this.painlessMethodKeysToImportedPainlessMethods = Collections.unmodifiableMap(painlessMethodKeysToImportedPainlessMethods); this.painlessMethodKeysToPainlessBindings = Collections.unmodifiableMap(painlessMethodKeysToPainlessBindings); } @@ -167,6 +174,14 @@ public PainlessField lookupPainlessField(Class targetClass, boolean isStatic, return painlessField; } + public PainlessMethod lookupImportedPainlessMethod(String methodName, int arity) { + Objects.requireNonNull(methodName); + + String painlessMethodKey = buildPainlessMethodKey(methodName, arity); + + return painlessMethodKeysToImportedPainlessMethods.get(painlessMethodKey); + } + public PainlessBinding lookupPainlessBinding(String methodName, int arity) { Objects.requireNonNull(methodName); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java index a64814f866113..b822bd47c7a48 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java @@ -243,6 +243,14 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { } } + for (WhitelistMethod whitelistStatic : whitelist.whitelistImportedMethods) { + origin = whitelistStatic.origin; + painlessLookupBuilder.addImportedPainlessMethod( + whitelist.classLoader, whitelistStatic.augmentedCanonicalClassName, + whitelistStatic.methodName, whitelistStatic.returnCanonicalTypeName, + whitelistStatic.canonicalTypeNameParameters); + } + for (WhitelistBinding whitelistBinding : whitelist.whitelistBindings) { origin = whitelistBinding.origin; painlessLookupBuilder.addPainlessBinding( @@ -261,12 +269,14 @@ public static PainlessLookup buildFromWhitelists(List whitelists) { private final Map> canonicalClassNamesToClasses; private final Map, PainlessClassBuilder> classesToPainlessClassBuilders; + private final Map painlessMethodKeysToImportedPainlessMethods; private final Map painlessMethodKeysToPainlessBindings; public PainlessLookupBuilder() { canonicalClassNamesToClasses = new HashMap<>(); classesToPainlessClassBuilders = new HashMap<>(); + painlessMethodKeysToImportedPainlessMethods = new HashMap<>(); painlessMethodKeysToPainlessBindings = new HashMap<>(); } @@ -513,8 +523,9 @@ public void addPainlessMethod(ClassLoader classLoader, String targetCanonicalCla addPainlessMethod(targetClass, augmentedClass, methodName, returnType, typeParameters); } - public void addPainlessMethod(Class targetClass, Class augmentedClass, String methodName, - Class returnType, List> typeParameters) { + public void addPainlessMethod(Class targetClass, Class augmentedClass, + String methodName, Class returnType, List> typeParameters) { + Objects.requireNonNull(targetClass); Objects.requireNonNull(methodName); Objects.requireNonNull(returnType); @@ -573,6 +584,12 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, Str } else { try { javaMethod = augmentedClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); + + if (Modifier.isStatic(javaMethod.getModifiers()) == false) { + throw new IllegalArgumentException("method [[" + targetCanonicalClassName + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "] with augmented class " + + "[" + typeToCanonicalTypeName(augmentedClass) + "] must be static"); + } } catch (NoSuchMethodException nsme) { throw new IllegalArgumentException("method reflection object [[" + targetCanonicalClassName + "], " + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found " + @@ -620,7 +637,7 @@ public void addPainlessMethod(Class targetClass, Class augmentedClass, Str "with the same arity and different return type or type parameters"); } } else { - PainlessMethod painlessMethod = painlessClassBuilder.staticMethods.get(painlessMethodKey); + PainlessMethod painlessMethod = painlessClassBuilder.methods.get(painlessMethodKey); if (painlessMethod == null) { MethodHandle methodHandle; @@ -788,6 +805,146 @@ public void addPainlessField(Class targetClass, String fieldName, Class ty } } + public void addImportedPainlessMethod(ClassLoader classLoader, String targetCanonicalClassName, + String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters) { + + Objects.requireNonNull(classLoader); + Objects.requireNonNull(targetCanonicalClassName); + Objects.requireNonNull(methodName); + Objects.requireNonNull(returnCanonicalTypeName); + Objects.requireNonNull(canonicalTypeNameParameters); + + Class targetClass = canonicalClassNamesToClasses.get(targetCanonicalClassName); + + if (targetClass == null) { + throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + List> typeParameters = new ArrayList<>(canonicalTypeNameParameters.size()); + + for (String canonicalTypeNameParameter : canonicalTypeNameParameters) { + Class typeParameter = canonicalTypeNameToType(canonicalTypeNameParameter); + + if (typeParameter == null) { + throw new IllegalArgumentException("type parameter [" + canonicalTypeNameParameter + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + typeParameters.add(typeParameter); + } + + Class returnType = canonicalTypeNameToType(returnCanonicalTypeName); + + if (returnType == null) { + throw new IllegalArgumentException("return type [" + returnCanonicalTypeName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + canonicalTypeNameParameters + "]"); + } + + addImportedPainlessMethod(targetClass, methodName, returnType, typeParameters); + } + + public void addImportedPainlessMethod(Class targetClass, String methodName, Class returnType, List> typeParameters) { + Objects.requireNonNull(targetClass); + Objects.requireNonNull(methodName); + Objects.requireNonNull(returnType); + Objects.requireNonNull(typeParameters); + + if (targetClass == def.class) { + throw new IllegalArgumentException("cannot add imported method from reserved class [" + DEF_CLASS_NAME + "]"); + } + + String targetCanonicalClassName = typeToCanonicalTypeName(targetClass); + + if (METHOD_NAME_PATTERN.matcher(methodName).matches() == false) { + throw new IllegalArgumentException( + "invalid imported method name [" + methodName + "] for target class [" + targetCanonicalClassName + "]."); + } + + PainlessClassBuilder painlessClassBuilder = classesToPainlessClassBuilders.get(targetClass); + + if (painlessClassBuilder == null) { + throw new IllegalArgumentException("target class [" + targetCanonicalClassName + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + int typeParametersSize = typeParameters.size(); + List> javaTypeParameters = new ArrayList<>(typeParametersSize); + + for (Class typeParameter : typeParameters) { + if (isValidType(typeParameter) == false) { + throw new IllegalArgumentException("type parameter [" + typeToCanonicalTypeName(typeParameter) + "] " + + "not found for imported method [[" + targetCanonicalClassName + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + javaTypeParameters.add(typeToJavaType(typeParameter)); + } + + if (isValidType(returnType) == false) { + throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(returnType) + "] not found for imported method " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + Method javaMethod; + + try { + javaMethod = targetClass.getMethod(methodName, javaTypeParameters.toArray(new Class[typeParametersSize])); + } catch (NoSuchMethodException nsme) { + throw new IllegalArgumentException("imported method reflection object [[" + targetCanonicalClassName + "], " + + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", nsme); + } + + if (javaMethod.getReturnType() != typeToJavaType(returnType)) { + throw new IllegalArgumentException("return type [" + typeToCanonicalTypeName(javaMethod.getReturnType()) + "] " + + "does not match the specified returned type [" + typeToCanonicalTypeName(returnType) + "] " + + "for imported method [[" + targetClass.getCanonicalName() + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "]"); + } + + if (Modifier.isStatic(javaMethod.getModifiers()) == false) { + throw new IllegalArgumentException("imported method [[" + targetClass.getCanonicalName() + "], [" + methodName + "], " + + typesToCanonicalTypeNames(typeParameters) + "] must be static"); + } + + String painlessMethodKey = buildPainlessMethodKey(methodName, typeParametersSize); + + if (painlessMethodKeysToPainlessBindings.containsKey(painlessMethodKey)) { + throw new IllegalArgumentException("imported method and binding cannot have the same name [" + methodName + "]"); + } + + PainlessMethod importedPainlessMethod = painlessMethodKeysToImportedPainlessMethods.get(painlessMethodKey); + + if (importedPainlessMethod == null) { + MethodHandle methodHandle; + + try { + methodHandle = MethodHandles.publicLookup().in(targetClass).unreflect(javaMethod); + } catch (IllegalAccessException iae) { + throw new IllegalArgumentException("imported method handle [[" + targetClass.getCanonicalName() + "], " + + "[" + methodName + "], " + typesToCanonicalTypeNames(typeParameters) + "] not found", iae); + } + + MethodType methodType = methodHandle.type(); + + importedPainlessMethod = painlessMethodCache.computeIfAbsent( + new PainlessMethodCacheKey(targetClass, methodName, returnType, typeParameters), + key -> new PainlessMethod(javaMethod, targetClass, returnType, typeParameters, methodHandle, methodType)); + + painlessMethodKeysToImportedPainlessMethods.put(painlessMethodKey, importedPainlessMethod); + } else if (importedPainlessMethod.returnType == returnType && + importedPainlessMethod.typeParameters.equals(typeParameters) == false) { + throw new IllegalArgumentException("cannot have imported methods " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + + "[" + typeToCanonicalTypeName(returnType) + "], " + + typesToCanonicalTypeNames(typeParameters) + "] and " + + "[[" + targetCanonicalClassName + "], [" + methodName + "], " + + "[" + typeToCanonicalTypeName(importedPainlessMethod.returnType) + "], " + + typesToCanonicalTypeNames(importedPainlessMethod.typeParameters) + "] " + + "with the same arity and different return type or type parameters"); + } + } + public void addPainlessBinding(ClassLoader classLoader, String targetJavaClassName, String methodName, String returnCanonicalTypeName, List canonicalTypeNameParameters) { @@ -937,6 +1094,11 @@ public void addPainlessBinding(Class targetClass, String methodName, Class } String painlessMethodKey = buildPainlessMethodKey(methodName, constructorTypeParametersSize + methodTypeParametersSize); + + if (painlessMethodKeysToImportedPainlessMethods.containsKey(painlessMethodKey)) { + throw new IllegalArgumentException("binding and imported method cannot have the same name [" + methodName + "]"); + } + PainlessBinding painlessBinding = painlessMethodKeysToPainlessBindings.get(painlessMethodKey); if (painlessBinding == null) { @@ -976,7 +1138,8 @@ public PainlessLookup build() { classesToPainlessClasses.put(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue().build()); } - return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses, painlessMethodKeysToPainlessBindings); + return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses, + painlessMethodKeysToImportedPainlessMethods, painlessMethodKeysToPainlessBindings); } private void copyPainlessClassMembers() { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java index 8ae6ad9723da4..d161296d90a56 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java @@ -25,6 +25,7 @@ import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.lookup.PainlessBinding; +import org.elasticsearch.painless.lookup.PainlessMethod; import org.objectweb.asm.Label; import org.objectweb.asm.Type; import org.objectweb.asm.commons.Method; @@ -45,6 +46,7 @@ public final class ECallLocal extends AExpression { private final List arguments; private LocalMethod method = null; + private PainlessMethod imported = null; private PainlessBinding binding = null; public ECallLocal(Location location, String name, List arguments) { @@ -65,16 +67,33 @@ void extractVariables(Set variables) { void analyze(Locals locals) { method = locals.getMethod(name, arguments.size()); - if (method == null) { - binding = locals.getPainlessLookup().lookupPainlessBinding(name, arguments.size()); + imported = locals.getPainlessLookup().lookupImportedPainlessMethod(name, arguments.size()); + + if (imported == null) { + binding = locals.getPainlessLookup().lookupPainlessBinding(name, arguments.size()); - if (binding == null) { - throw createError(new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments.")); + if (binding == null) { + throw createError( + new IllegalArgumentException("Unknown call [" + name + "] with [" + arguments.size() + "] arguments.")); + } } } - List> typeParameters = new ArrayList<>(method == null ? binding.typeParameters : method.typeParameters); + List> typeParameters; + + if (method != null) { + typeParameters = new ArrayList<>(method.typeParameters); + actual = method.returnType; + } else if (imported != null) { + typeParameters = new ArrayList<>(imported.typeParameters); + actual = imported.returnType; + } else if (binding != null) { + typeParameters = new ArrayList<>(binding.typeParameters); + actual = binding.returnType; + } else { + throw new IllegalStateException("Illegal tree structure."); + } for (int argument = 0; argument < arguments.size(); ++argument) { AExpression expression = arguments.get(argument); @@ -86,14 +105,26 @@ void analyze(Locals locals) { } statement = true; - actual = method == null ? binding.returnType : method.returnType; } @Override void write(MethodWriter writer, Globals globals) { writer.writeDebugInfo(location); - if (method == null) { + if (method != null) { + for (AExpression argument : arguments) { + argument.write(writer, globals); + } + + writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString())); + } else if (imported != null) { + for (AExpression argument : arguments) { + argument.write(writer, globals); + } + + writer.invokeStatic(Type.getType(imported.targetClass), + new Method(imported.javaMethod.getName(), imported.methodType.toMethodDescriptorString())); + } else if (binding != null) { String name = globals.addBinding(binding.javaConstructor.getDeclaringClass()); Type type = Type.getType(binding.javaConstructor.getDeclaringClass()); int javaConstructorParameterCount = binding.javaConstructor.getParameterCount(); @@ -124,11 +155,7 @@ void write(MethodWriter writer, Globals globals) { writer.invokeVirtual(type, Method.getMethod(binding.javaMethod)); } else { - for (AExpression argument : arguments) { - argument.write(writer, globals); - } - - writer.invokeStatic(CLASS_TYPE, new Method(method.name, method.methodType.toMethodDescriptorString())); + throw new IllegalStateException("Illegal tree structure."); } } diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt index 444234384c6d3..81009de9979eb 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt @@ -176,6 +176,7 @@ class org.elasticsearch.painless.FeatureTest no_import { } # for testing -static { +static_import { + float staticAddFloatsTest(float, float) from_class org.elasticsearch.painless.FeatureTest int testAddWithState(int, int, int, double) bound_to org.elasticsearch.painless.BindingTest } \ No newline at end of file diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java index 25866c8d668a3..9863db0b21eac 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicAPITests.java @@ -133,4 +133,8 @@ public void testPublicMemberAccess() { public void testNoSemicolon() { assertEquals(true, exec("def x = true; if (x) return x")); } + + public void testStatic() { + assertEquals(15.5f, exec("staticAddFloatsTest(6.5f, 9.0f)")); + } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index 96cc296a1af52..963a433f172e8 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -47,6 +47,8 @@ * Typically just asserts the output of {@code exec()} */ public abstract class ScriptTestCase extends ESTestCase { + private static final PainlessLookup PAINLESS_LOOKUP = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS); + protected PainlessScriptEngine scriptEngine; @Before @@ -92,12 +94,12 @@ public Object exec(String script, Map vars, boolean picky) { public Object exec(String script, Map vars, Map compileParams, Scorer scorer, boolean picky) { // test for ambiguity errors before running the actual script if picky is true if (picky) { - PainlessLookup painlessLookup = PainlessLookupBuilder.buildFromWhitelists(Whitelist.BASE_WHITELISTS); - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class); + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(PAINLESS_LOOKUP, GenericElasticsearchScript.class); CompilerSettings pickySettings = new CompilerSettings(); pickySettings.setPicky(true); pickySettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(scriptEngineSettings())); - Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, painlessLookup, null); + Walker.buildPainlessTree( + scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, PAINLESS_LOOKUP, null); } // test actual script execution ExecutableScript.Factory factory = scriptEngine.compile(null, script, ExecutableScript.CONTEXT, compileParams); From 0b62d6bb9d7a441ef6802d43dcd4e2452961498f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 7 Sep 2018 21:27:59 -0400 Subject: [PATCH 02/12] CRUD: AwaitsFix entire wait_for_refresh close test I disabled one branch a few hours ago because it failed in CI. It looks like other branches can also fail so I'll disable them as well and look more closely on Monday. --- .../org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java index 09d6b9e51a5fc..ffd3a1f6c0c3c 100644 --- a/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java +++ b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseIT.java @@ -38,6 +38,7 @@ /** * Tests that wait for refresh is fired if the index is closed. */ +@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33533") public class WaitForRefreshAndCloseIT extends ESRestTestCase { @Before public void setupIndex() throws IOException { @@ -76,7 +77,6 @@ public void testUpdateAndThenClose() throws Exception { closeWhileListenerEngaged(start("POST", "/_update", "{\"doc\":{\"name\":\"test\"}}")); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/33533") public void testDeleteAndThenClose() throws Exception { Request request = new Request("PUT", docPath()); request.setJsonEntity("{\"test\":\"test\"}"); From 97736ac46a54073cf7081e9ed08bccbdac79ea6a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 7 Sep 2018 21:42:58 -0400 Subject: [PATCH 03/12] Logging: Skip test if it'd fail If we're running on a platform where we can't install syscall filters Elasticsearch logs a message before it reads the data directory to get the node name. Because that log message doesn't have a node name this test will fail. Since we mostly run the test on OSes where we *can* install the syscall filters we can fairly safely skip the test on OSes where we can't install the syscall filters. Closes #33540 --- .../elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java index 512fc2345549c..76bc9fa4d55fb 100644 --- a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java +++ b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.unconfigured_node_name; +import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.common.logging.NodeNameInLogsIntegTestCase; import java.io.IOException; @@ -32,6 +33,8 @@ public class NodeNameInLogsIT extends NodeNameInLogsIntegTestCase { @Override protected BufferedReader openReader(Path logFile) throws IOException { + assumeTrue("We log a line without the node name if we can't install the seccomp filters", + BootstrapInfo.isSystemCallFilterInstalled()); return AccessController.doPrivileged((PrivilegedAction) () -> { try { return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); From ab9e2cddf169be536065e9b32d367fee978bbb0c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 7 Sep 2018 21:54:38 -0400 Subject: [PATCH 04/12] Logging: Clean up skipping test Clean up on top of the last fix: if we skip the entire test case then the test run would fail because we skipped all the tests. This adds a dummy test case to prevent that. It is a fairly nasty work around I plan to work on something that makes this not required any more anyway. --- .../unconfigured_node_name/NodeNameInLogsIT.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java index 76bc9fa4d55fb..9f36a600b68de 100644 --- a/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java +++ b/qa/unconfigured-node-name/src/test/java/org/elasticsearch/unconfigured_node_name/NodeNameInLogsIT.java @@ -43,4 +43,11 @@ protected BufferedReader openReader(Path logFile) throws IOException { } }); } + + public void testDummy() { + /* Dummy test case so that when we run this test on a platform that + * does not support our syscall filters and we skip the test above + * we don't fail the entire test run because we skipped all the tests. + */ + } } From f27c3dcf881d8dc426d6d58afb63fb3774f45479 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sat, 8 Sep 2018 10:18:45 +0200 Subject: [PATCH 05/12] INGEST: Remove Outdated TODOs (#33458) * CompoundProcessor is in the ingest package now -> resolved * Java generics don't offer type checking so nothing can be done here -> remvoed TODO and test * #16019 was closed and not acted on -> todo can go away --- .../java/org/elasticsearch/ingest/CompoundProcessor.java | 1 - .../org/elasticsearch/ingest/ConfigurationUtilsTests.java | 6 ------ .../test/java/org/elasticsearch/ingest/IngestClientIT.java | 1 - 3 files changed, 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index f576667f44109..e1a413f6aa9bb 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -16,7 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -//TODO(simonw): can all these classes go into org.elasticsearch.ingest? package org.elasticsearch.ingest; diff --git a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index f3a11a86e54e5..9111658e49ca8 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -92,12 +92,6 @@ public void testReadBooleanPropertyInvalidType() { } } - // TODO(talevy): Issue with generics. This test should fail, "int" is of type List - public void testOptional_InvalidType() { - List val = ConfigurationUtils.readList(null, null, config, "int"); - assertThat(val, equalTo(Collections.singletonList(2))); - } - public void testReadStringOrIntProperty() { String val1 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "foo", null); String val2 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "num", null); diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 65139109a83a2..6e5d862372ac6 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -60,7 +60,6 @@ public class IngestClientIT extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { - // TODO: Remove this method once gets in: https://github.com/elastic/elasticsearch/issues/16019 if (nodeOrdinal % 2 == 0) { return Settings.builder().put("node.ingest", false).put(super.nodeSettings(nodeOrdinal)).build(); } From 94e4cb64c2480141fd54e19f64291ad7cd870bef Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Sat, 8 Sep 2018 19:29:31 -0400 Subject: [PATCH 06/12] Bootstrap a new history_uuid when force allocating a stale primary (#33432) This commit ensures that we bootstrap a new history_uuid when force allocating a stale primary. A stale primary should never be the source of an operation-based recovery to another shard which exists before the forced-allocation. Closes #26712 --- .../cluster/routing/IndexRoutingTable.java | 9 ++- .../cluster/routing/RecoverySource.java | 80 +++++++++++++++---- .../cluster/routing/ShardRouting.java | 5 +- ...AllocateEmptyPrimaryAllocationCommand.java | 4 +- ...AllocateStalePrimaryAllocationCommand.java | 3 +- .../index/shard/StoreRecovery.java | 3 + .../action/search/SearchAsyncActionTests.java | 2 +- .../cluster/routing/AllocationIdTests.java | 10 +-- .../routing/GroupShardsIteratorTests.java | 2 +- .../cluster/routing/PrimaryAllocationIT.java | 39 ++++++--- .../allocation/ThrottlingAllocationTests.java | 2 +- .../DiskThresholdDeciderUnitTests.java | 12 +-- .../index/shard/IndexShardIT.java | 2 +- .../index/shard/IndexShardTests.java | 55 +++++++++---- ...dicesLifecycleListenerSingleNodeTests.java | 2 +- .../indices/recovery/IndexRecoveryIT.java | 7 +- .../BlobStoreRepositoryRestoreTests.java | 5 +- .../action/cat/RestIndicesActionTests.java | 4 +- .../cluster/routing/TestShardRouting.java | 8 +- .../ESIndexLevelReplicationTestCase.java | 2 +- .../index/shard/IndexShardTestCase.java | 8 +- .../action/TransportOpenJobActionTests.java | 4 +- .../datafeed/DatafeedNodeSelectorTests.java | 2 +- .../ClusterStatsMonitoringDocTests.java | 5 +- .../authz/store/NativeRolesStoreTests.java | 6 +- .../support/SecurityIndexManagerTests.java | 9 ++- .../security/test/SecurityTestUtils.java | 68 +--------------- 27 files changed, 194 insertions(+), 164 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 4e7e81def8739..e9d805d34c8a1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -27,10 +27,11 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -386,7 +387,7 @@ private Builder initializeAsRestore(IndexMetaData indexMetaData, SnapshotRecover if (asNew && ignoreShards.contains(shardNumber)) { // This shards wasn't completely snapshotted - restore it as new shard indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, - primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); + primary ? EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, unassignedInfo)); } else { indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(shardId, primary, primary ? recoverySource : PeerRecoverySource.INSTANCE, unassignedInfo)); @@ -410,13 +411,13 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas final RecoverySource primaryRecoverySource; if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) { // we have previous valid copies for this shard. use them for recovery - primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; + primaryRecoverySource = ExistingStoreRecoverySource.INSTANCE; } else if (indexMetaData.getResizeSourceIndex() != null) { // this is a new index but the initial shards should merged from another index primaryRecoverySource = LocalShardsRecoverySource.INSTANCE; } else { // a freshly created index with no restriction - primaryRecoverySource = StoreRecoverySource.EMPTY_STORE_INSTANCE; + primaryRecoverySource = EmptyStoreRecoverySource.INSTANCE; } IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId); for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java index 13cb85ea399d5..2502fb0f3cc62 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RecoverySource.java @@ -34,7 +34,8 @@ /** * Represents the recovery source of a shard. Available recovery types are: * - * - {@link StoreRecoverySource} recovery from the local store (empty or with existing data) + * - {@link EmptyStoreRecoverySource} recovery from an empty store + * - {@link ExistingStoreRecoverySource} recovery from an existing store * - {@link PeerRecoverySource} recovery from a primary on another node * - {@link SnapshotRecoverySource} recovery from a snapshot * - {@link LocalShardsRecoverySource} recovery from other shards of another index on the same node @@ -59,8 +60,8 @@ public void addAdditionalFields(XContentBuilder builder, ToXContent.Params param public static RecoverySource readFrom(StreamInput in) throws IOException { Type type = Type.values()[in.readByte()]; switch (type) { - case EMPTY_STORE: return StoreRecoverySource.EMPTY_STORE_INSTANCE; - case EXISTING_STORE: return StoreRecoverySource.EXISTING_STORE_INSTANCE; + case EMPTY_STORE: return EmptyStoreRecoverySource.INSTANCE; + case EXISTING_STORE: return new ExistingStoreRecoverySource(in); case PEER: return PeerRecoverySource.INSTANCE; case SNAPSHOT: return new SnapshotRecoverySource(in); case LOCAL_SHARDS: return LocalShardsRecoverySource.INSTANCE; @@ -91,6 +92,10 @@ public enum Type { public abstract Type getType(); + public boolean shouldBootstrapNewHistoryUUID() { + return false; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -107,25 +112,68 @@ public int hashCode() { } /** - * recovery from an existing on-disk store or a fresh copy + * Recovery from a fresh copy */ - public abstract static class StoreRecoverySource extends RecoverySource { - public static final StoreRecoverySource EMPTY_STORE_INSTANCE = new StoreRecoverySource() { - @Override - public Type getType() { - return Type.EMPTY_STORE; + public static final class EmptyStoreRecoverySource extends RecoverySource { + public static final EmptyStoreRecoverySource INSTANCE = new EmptyStoreRecoverySource(); + + @Override + public Type getType() { + return Type.EMPTY_STORE; + } + + @Override + public String toString() { + return "new shard recovery"; + } + } + + /** + * Recovery from an existing on-disk store + */ + public static final class ExistingStoreRecoverySource extends RecoverySource { + public static final ExistingStoreRecoverySource INSTANCE = new ExistingStoreRecoverySource(false); + public static final ExistingStoreRecoverySource FORCE_STALE_PRIMARY_INSTANCE = new ExistingStoreRecoverySource(true); + + private final boolean bootstrapNewHistoryUUID; + + private ExistingStoreRecoverySource(boolean bootstrapNewHistoryUUID) { + this.bootstrapNewHistoryUUID = bootstrapNewHistoryUUID; + } + + private ExistingStoreRecoverySource(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + bootstrapNewHistoryUUID = in.readBoolean(); + } else { + bootstrapNewHistoryUUID = false; } - }; - public static final StoreRecoverySource EXISTING_STORE_INSTANCE = new StoreRecoverySource() { - @Override - public Type getType() { - return Type.EXISTING_STORE; + } + + @Override + public void addAdditionalFields(XContentBuilder builder, Params params) throws IOException { + builder.field("bootstrap_new_history_uuid", bootstrapNewHistoryUUID); + } + + @Override + protected void writeAdditionalFields(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeBoolean(bootstrapNewHistoryUUID); } - }; + } + + @Override + public boolean shouldBootstrapNewHistoryUUID() { + return bootstrapNewHistoryUUID; + } + + @Override + public Type getType() { + return Type.EXISTING_STORE; + } @Override public String toString() { - return getType() == Type.EMPTY_STORE ? "new shard recovery" : "existing recovery"; + return "existing store recovery; bootstrap_history_uuid=" + bootstrapNewHistoryUUID; } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java index 6a9a105b6c432..74341ca271a9c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/ShardRouting.java @@ -19,14 +19,13 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.Index; @@ -318,7 +317,7 @@ public ShardRouting moveToUnassigned(UnassignedInfo unassignedInfo) { final RecoverySource recoverySource; if (active()) { if (primary()) { - recoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; + recoverySource = ExistingStoreRecoverySource.INSTANCE; } else { recoverySource = PeerRecoverySource.INSTANCE; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java index 66281b73458b3..a42fd2765b598 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateEmptyPrimaryAllocationCommand.java @@ -21,7 +21,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.ShardRouting; @@ -136,7 +136,7 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) } initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, unassignedInfoToUpdate, - StoreRecoverySource.EMPTY_STORE_INSTANCE); + EmptyStoreRecoverySource.INSTANCE); return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders")); } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java index 11c4420200e33..f4c9aba17d71e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/command/AllocateStalePrimaryAllocationCommand.java @@ -129,7 +129,8 @@ public RerouteExplanation execute(RoutingAllocation allocation, boolean explain) "trying to allocate an existing primary shard [" + index + "][" + shardId + "], while no such shard has ever been active"); } - initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting); + initializeUnassignedShard(allocation, routingNodes, routingNode, shardRouting, null, + RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE); return new RerouteExplanation(this, allocation.decision(Decision.YES, name() + " (allocation command)", "ignore deciders")); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index ae3f90e63e7d1..c4b971e470d66 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -398,6 +398,9 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); } else if (indexShouldExists) { + if (recoveryState.getRecoverySource().shouldBootstrapNewHistoryUUID()) { + store.bootstrapNewHistory(); + } // since we recover from local, just fill the files and size try { final RecoveryState.Index index = recoveryState.getIndex(); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java index 1f7f6f4249b0c..95282e358e144 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchAsyncActionTests.java @@ -377,7 +377,7 @@ static GroupShardsIterator getShardsIter(String index, Orig ArrayList unassigned = new ArrayList<>(); ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")); + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar")); routing = routing.initialize(primaryNode.getId(), i + "p", 0); routing.started(); started.add(routing); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java index c1861572d8352..86dbeabd1d73e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/AllocationIdTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.cluster.routing; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; @@ -37,7 +37,7 @@ public class AllocationIdTests extends ESTestCase { public void testShardToStarted() { logger.info("-- create unassigned shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); assertThat(shard.allocationId(), nullValue()); logger.info("-- initialize the shard"); @@ -57,7 +57,7 @@ public void testShardToStarted() { public void testSuccessfulRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); @@ -80,7 +80,7 @@ public void testSuccessfulRelocation() { public void testCancelRelocation() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); @@ -100,7 +100,7 @@ public void testCancelRelocation() { public void testMoveToUnassigned() { logger.info("-- build started shard"); - ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, StoreRecoverySource.EXISTING_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); + ShardRouting shard = ShardRouting.newUnassigned(new ShardId("test","_na_", 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)); shard = shard.initialize("node1", null, -1); shard = shard.moveToStarted(); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java index f2571fce3391d..66eabd4cbd921 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/GroupShardsIteratorTests.java @@ -77,7 +77,7 @@ public void testIterate() { public ShardRouting newRouting(Index index, int id, boolean started) { ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, id), true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); shardRouting = ShardRoutingHelper.initialize(shardRouting, "some node"); if (started) { shardRouting = ShardRoutingHelper.moveToStarted(shardRouting); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java index 9786c0eaf5290..9b2db5b34b1da 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.gateway.GatewayAllocator; +import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.EngineTestCase; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; @@ -55,6 +56,7 @@ import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_SHARDS; @@ -64,6 +66,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.isIn; import static org.hamcrest.Matchers.not; @@ -83,18 +86,9 @@ protected Settings nodeSettings(int nodeOrdinal) { .put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build(); } - private void createStaleReplicaScenario() throws Exception { - logger.info("--> starting 3 nodes, 1 master, 2 data"); - String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); - internalCluster().startDataOnlyNodes(2); - - assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder() - .put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); - ensureGreen(); - logger.info("--> indexing..."); + private void createStaleReplicaScenario(String master) throws Exception { client().prepareIndex("test", "type1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get(); refresh(); - ClusterState state = client().admin().cluster().prepareState().all().get().getState(); List shards = state.routingTable().allShards("test"); assertThat(shards.size(), equalTo(2)); @@ -140,7 +134,13 @@ private void createStaleReplicaScenario() throws Exception { } public void testDoNotAllowStaleReplicasToBePromotedToPrimary() throws Exception { - createStaleReplicaScenario(); + logger.info("--> starting 3 nodes, 1 master, 2 data"); + String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodes(2); + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); + ensureGreen(); + createStaleReplicaScenario(master); logger.info("--> starting node that reuses data folder with the up-to-date primary shard"); internalCluster().startDataOnlyNode(Settings.EMPTY); @@ -176,9 +176,17 @@ public void testFailedAllocationOfStalePrimaryToDataNodeWithNoData() throws Exce } public void testForceStaleReplicaToBePromotedToPrimary() throws Exception { - boolean useStaleReplica = randomBoolean(); // if true, use stale replica, otherwise a completely empty copy - createStaleReplicaScenario(); + logger.info("--> starting 3 nodes, 1 master, 2 data"); + String master = internalCluster().startMasterOnlyNode(Settings.EMPTY); + internalCluster().startDataOnlyNodes(2); + assertAcked(client().admin().indices().prepareCreate("test") + .setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1)).get()); + ensureGreen(); + Set historyUUIDs = Arrays.stream(client().admin().indices().prepareStats("test").clear().get().getShards()) + .map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet()); + createStaleReplicaScenario(master); + boolean useStaleReplica = randomBoolean(); // if true, use stale replica, otherwise a completely empty copy logger.info("--> explicitly promote old primary shard"); final String idxName = "test"; ImmutableOpenIntMap> storeStatuses = client().admin().indices().prepareShardStores(idxName).get().getStoreStatuses().get(idxName); @@ -213,6 +221,11 @@ public void testForceStaleReplicaToBePromotedToPrimary() throws Exception { ClusterState state = client().admin().cluster().prepareState().get().getState(); assertEquals(Collections.singleton(state.routingTable().index(idxName).shard(0).primary.allocationId().getId()), state.metaData().index(idxName).inSyncAllocationIds(0)); + + Set newHistoryUUIds = Arrays.stream(client().admin().indices().prepareStats("test").clear().get().getShards()) + .map(shard -> shard.getCommitStats().getUserData().get(Engine.HISTORY_UUID_KEY)).collect(Collectors.toSet()); + assertThat(newHistoryUUIds, everyItem(not(isIn(historyUUIDs)))); + assertThat(newHistoryUUIds, hasSize(1)); } public void testForcePrimaryShardIfAllocationDecidersSayNoAfterIndexCreation() throws ExecutionException, InterruptedException { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java index d32ebe62ec1aa..01586d9c49575 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/ThrottlingAllocationTests.java @@ -399,7 +399,7 @@ private void addInSyncAllocationIds(Index index, IndexMetaData.Builder indexMeta final boolean primary = randomBoolean(); final ShardRouting unassigned = ShardRouting.newUnassigned(new ShardId(index, shard), primary, primary ? - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : + RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "test") ); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java index da0e0a9b0bcfa..ce53c14807c22 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDeciderUnitTests.java @@ -29,9 +29,9 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource.EmptyStoreRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.LocalShardsRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -69,7 +69,7 @@ public void testCanAllocateUsesMaxAvailableSpace() { final Index index = metaData.index("test").getIndex(); - ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(), new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT); DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(), @@ -125,22 +125,22 @@ public void testCanRemainUsesLeastAvailableSpace() { .build(); final IndexMetaData indexMetaData = metaData.index("test"); - ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 0), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_0 = ShardRoutingHelper.initialize(test_0, node_0.getId()); test_0 = ShardRoutingHelper.moveToStarted(test_0); shardRoutingMap.put(test_0, "/node0/least"); - ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_1 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 1), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_1 = ShardRoutingHelper.initialize(test_1, node_1.getId()); test_1 = ShardRoutingHelper.moveToStarted(test_1); shardRoutingMap.put(test_1, "/node1/least"); - ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_2 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 2), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_2 = ShardRoutingHelper.initialize(test_2, node_1.getId()); test_2 = ShardRoutingHelper.moveToStarted(test_2); shardRoutingMap.put(test_2, "/node1/most"); - ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, StoreRecoverySource.EMPTY_STORE_INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); + ShardRouting test_3 = ShardRouting.newUnassigned(new ShardId(indexMetaData.getIndex(), 3), true, EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo")); test_3 = ShardRoutingHelper.initialize(test_3, node_1.getId()); test_3 = ShardRoutingHelper.moveToStarted(test_3); // Intentionally not in the shardRoutingMap. We want to test what happens when we don't know where it is. diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index 182747e7dda5d..87edfcfccb150 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -637,7 +637,7 @@ private static ShardRouting getInitializingShardRouting(ShardRouting existingSha existingShardRouting.currentNodeId(), null, existingShardRouting.primary(), ShardRoutingState.INITIALIZING, existingShardRouting.allocationId()); shardRouting = shardRouting.updateUnassigned(new UnassignedInfo(UnassignedInfo.Reason.INDEX_REOPENED, "fake recovery"), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + RecoverySource.ExistingStoreRecoverySource.INSTANCE); return shardRouting; } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 713bc04634b0a..4ed74388f0e1e 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -178,6 +178,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -843,7 +844,7 @@ public void testGlobalCheckpointSync() throws IOException { randomAlphaOfLength(8), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + RecoverySource.EmptyStoreRecoverySource.INSTANCE); final Settings settings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2) @@ -1199,7 +1200,7 @@ public void testShardStats() throws IOException { public void testShardStatsWithFailures() throws IOException { allowShardFailures(); final ShardId shardId = new ShardId("index", "_na_", 0); - final ShardRouting shardRouting = newShardRouting(shardId, "node", true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, ShardRoutingState.INITIALIZING); + final ShardRouting shardRouting = newShardRouting(shardId, "node", true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, ShardRoutingState.INITIALIZING); final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir()); @@ -1659,7 +1660,7 @@ public void testRecoverFromStoreWithOutOfOrderDelete() throws IOException { final ShardRouting replicaRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); @@ -1684,6 +1685,7 @@ public void testRecoverFromStore() throws IOException { flushShard(shard); translogOps = 0; } + String historyUUID = shard.getHistoryUUID(); IndexShard newShard = reinitShard(shard); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); @@ -1698,6 +1700,29 @@ public void testRecoverFromStore() throws IOException { assertThat(newShard.getReplicationTracker().getTrackedLocalCheckpointForShard(newShard.routingEntry().allocationId().getId()) .getLocalCheckpoint(), equalTo(totalOps - 1L)); assertDocCount(newShard, totalOps); + assertThat(newShard.getHistoryUUID(), equalTo(historyUUID)); + closeShards(newShard); + } + + public void testRecoverFromStalePrimaryForceNewHistoryUUID() throws IOException { + final IndexShard shard = newStartedShard(true); + int totalOps = randomInt(10); + for (int i = 0; i < totalOps; i++) { + indexDoc(shard, "_doc", Integer.toString(i)); + } + if (randomBoolean()) { + shard.updateLocalCheckpointForShard(shard.shardRouting.allocationId().getId(), totalOps - 1); + flushShard(shard); + } + String historyUUID = shard.getHistoryUUID(); + IndexShard newShard = reinitShard(shard, newShardRouting(shard.shardId(), shard.shardRouting.currentNodeId(), true, + ShardRoutingState.INITIALIZING, RecoverySource.ExistingStoreRecoverySource.FORCE_STALE_PRIMARY_INSTANCE)); + DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); + newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); + assertTrue(newShard.recoverFromStore()); + IndexShardTestCase.updateRoutingEntry(newShard, newShard.routingEntry().moveToStarted()); + assertDocCount(newShard, totalOps); + assertThat(newShard.getHistoryUUID(), not(equalTo(historyUUID))); closeShards(newShard); } @@ -1734,7 +1759,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException { final ShardRouting primaryShardRouting = shard.routingEntry(); IndexShard newShard = reinitShard(otherShard, ShardRoutingHelper.initWithSameId(primaryShardRouting, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); @@ -1760,7 +1785,7 @@ public void testRecoverFromStoreWithNoOps() throws IOException { for (int i = 0; i < 2; i++) { newShard = reinitShard(newShard, ShardRoutingHelper.initWithSameId(primaryShardRouting, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue(newShard.recoverFromStore()); try (Translog.Snapshot snapshot = getTranslog(newShard).newSnapshot()) { @@ -1778,7 +1803,7 @@ public void testRecoverFromCleanStore() throws IOException { } final ShardRouting shardRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, - ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE) + ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.EmptyStoreRecoverySource.INSTANCE) ); DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -1827,7 +1852,7 @@ public void testFailIfIndexNotPresentInRecoverFromStore() throws Exception { } newShard = reinitShard(newShard, - ShardRoutingHelper.initWithSameId(routing, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE)); + ShardRoutingHelper.initWithSameId(routing, RecoverySource.EmptyStoreRecoverySource.INSTANCE)); newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null)); assertTrue("recover even if there is nothing to recover", newShard.recoverFromStore()); @@ -1865,7 +1890,7 @@ public void testRecoverFromStoreRemoveStaleOperations() throws Exception { final ShardRouting replicaRouting = shard.routingEntry(); IndexShard newShard = reinitShard(shard, newShardRouting(replicaRouting.shardId(), replicaRouting.currentNodeId(), true, ShardRoutingState.INITIALIZING, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE)); + RecoverySource.ExistingStoreRecoverySource.INSTANCE)); newShard.pendingPrimaryTerm++; newShard.operationPrimaryTerm++; DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT); @@ -1905,7 +1930,7 @@ public void testRestoreShard() throws IOException { assertDocs(target, "1"); flushShard(source); // only flush source ShardRouting routing = ShardRoutingHelper.initWithSameId(target.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + RecoverySource.ExistingStoreRecoverySource.INSTANCE); final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID())); routing = ShardRoutingHelper.newWithRestoreSource(routing, new RecoverySource.SnapshotRecoverySource(snapshot, Version.CURRENT, "test")); @@ -1974,7 +1999,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { }; closeShards(shard); IndexShard newShard = newShard( - ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE), + ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE), shard.shardPath(), shard.indexSettings().getIndexMetaData(), null, @@ -2127,7 +2152,7 @@ public IndexSearcher wrap(IndexSearcher searcher) throws EngineException { closeShards(shard); IndexShard newShard = newShard( - ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE), + ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE), shard.shardPath(), shard.indexSettings().getIndexMetaData(), null, @@ -2625,7 +2650,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO assertThat("corruption marker should not be there", corruptedMarkerCount.get(), equalTo(0)); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE + RecoverySource.ExistingStoreRecoverySource.INSTANCE ); // start shard and perform index check on startup. It enforce shard to fail due to corrupted index files final IndexMetaData indexMetaData = IndexMetaData.builder(indexShard.indexSettings().getIndexMetaData()) @@ -2666,7 +2691,7 @@ public void testShardDoesNotStartIfCorruptedMarkerIsPresent() throws Exception { final ShardPath shardPath = indexShard.shardPath(); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE + RecoverySource.ExistingStoreRecoverySource.INSTANCE ); final IndexMetaData indexMetaData = indexShard.indexSettings().getIndexMetaData(); @@ -2751,7 +2776,7 @@ public void testReadSnapshotAndCheckIndexConcurrently() throws Exception { closeShards(indexShard); final ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(indexShard.routingEntry(), - isPrimary ? RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE + isPrimary ? RecoverySource.ExistingStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE ); final IndexMetaData indexMetaData = IndexMetaData.builder(indexShard.indexSettings().getIndexMetaData()) .settings(Settings.builder() @@ -3261,7 +3286,7 @@ public void testFlushOnInactive() throws Exception { .settings(settings) .primaryTerm(0, 1).build(); ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(metaData.getIndex(), 0), "n1", true, ShardRoutingState - .INITIALIZING, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + .INITIALIZING, RecoverySource.EmptyStoreRecoverySource.INSTANCE); final ShardId shardId = shardRouting.shardId(); final NodeEnvironment.NodePath nodePath = new NodeEnvironment.NodePath(createTempDir()); ShardPath shardPath = new ShardPath(false, nodePath.resolve(shardId), nodePath.resolve(shardId), shardId); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java index aa06f9e9b7dfe..769cdfc8a9b53 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesLifecycleListenerSingleNodeTests.java @@ -128,7 +128,7 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem String nodeId = newRouting.currentNodeId(); UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom"); newRouting = newRouting.moveToUnassigned(unassignedInfo) - .updateUnassigned(unassignedInfo, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE); + .updateUnassigned(unassignedInfo, RecoverySource.EmptyStoreRecoverySource.INSTANCE); newRouting = ShardRoutingHelper.initialize(newRouting, nodeId); IndexShard shard = index.createShard(newRouting, s -> {}); IndexShardTestCase.updateRoutingEntry(shard, newRouting); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 213aa644665dd..6a6970675eb9b 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.SnapshotRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -186,7 +185,7 @@ public void testGatewayRecovery() throws Exception { RecoveryState recoveryState = recoveryStates.get(0); - assertRecoveryState(recoveryState, 0, StoreRecoverySource.EXISTING_STORE_INSTANCE, true, Stage.DONE, null, node); + assertRecoveryState(recoveryState, 0, RecoverySource.ExistingStoreRecoverySource.INSTANCE, true, Stage.DONE, null, node); validateIndexRecoveryState(recoveryState.getIndex()); } @@ -239,7 +238,7 @@ public void testReplicaRecovery() throws Exception { // validate node A recovery RecoveryState nodeARecoveryState = nodeAResponses.get(0); - assertRecoveryState(nodeARecoveryState, 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA); + assertRecoveryState(nodeARecoveryState, 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryState.getIndex()); // validate node B recovery @@ -295,7 +294,7 @@ public void testRerouteRecovery() throws Exception { List nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates); assertThat(nodeBRecoveryStates.size(), equalTo(1)); - assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA); + assertRecoveryState(nodeARecoveryStates.get(0), 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, Stage.DONE, null, nodeA); validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex()); assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java index fa7de2d629112..ba3fa84a19641 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryRestoreTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.TestUtil; import org.elasticsearch.cluster.metadata.RepositoryMetaData; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingHelper; import org.elasticsearch.common.UUIDs; @@ -49,7 +50,6 @@ import java.util.Arrays; import java.util.List; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.hamcrest.Matchers.containsString; /** @@ -99,7 +99,8 @@ public void testRestoreSnapshotWithExistingFiles() throws IOException { } // build a new shard using the same store directory as the closed shard - ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), EXISTING_STORE_INSTANCE); + ShardRouting shardRouting = ShardRoutingHelper.initWithSameId(shard.routingEntry(), + RecoverySource.ExistingStoreRecoverySource.INSTANCE); shard = newShard( shardRouting, shard.shardPath(), diff --git a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java index cd592c9ed1e9c..4535bf7a91b0d 100644 --- a/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java +++ b/server/src/test/java/org/elasticsearch/rest/action/cat/RestIndicesActionTests.java @@ -30,8 +30,8 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RecoverySource.PeerRecoverySource; -import org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.Table; @@ -143,7 +143,7 @@ private IndicesStatsResponse randomIndicesStatsResponse(final Index[] indices) { boolean primary = (i == primaryIdx); Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(i)); ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, primary, - primary ? StoreRecoverySource.EMPTY_STORE_INSTANCE : PeerRecoverySource.INSTANCE, + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null) ); shardRouting = shardRouting.initialize("node-0", null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index 2291c3d39e200..c91c04884c5a7 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -88,8 +88,8 @@ private static RecoverySource buildRecoveryTarget(boolean primary, ShardRoutingS case UNASSIGNED: case INITIALIZING: if (primary) { - return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE); + return ESTestCase.randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, + RecoverySource.ExistingStoreRecoverySource.INSTANCE); } else { return RecoverySource.PeerRecoverySource.INSTANCE; } @@ -130,8 +130,8 @@ private static UnassignedInfo buildUnassignedInfo(ShardRoutingState state) { } public static RecoverySource randomRecoverySource() { - return ESTestCase.randomFrom(RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, - RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE, + return ESTestCase.randomFrom(RecoverySource.EmptyStoreRecoverySource.INSTANCE, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, RecoverySource.PeerRecoverySource.INSTANCE, RecoverySource.LocalShardsRecoverySource.INSTANCE, new RecoverySource.SnapshotRecoverySource( diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index f2afdff9c3a3a..8717d7ba146fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -172,7 +172,7 @@ protected ReplicationGroup(final IndexMetaData indexMetaData) throws IOException private ShardRouting createShardRouting(String nodeId, boolean primary) { return TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); } protected EngineFactory getEngineFactory(ShardRouting routing) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 53576a1d80a70..9082b4153b0bf 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -193,7 +193,7 @@ protected IndexShard newShard(final boolean primary, final Settings settings) th */ protected IndexShard newShard(boolean primary, Settings settings, EngineFactory engineFactory) throws IOException { final RecoverySource recoverySource = - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE; + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE; final ShardRouting shardRouting = TestShardRouting.newShardRouting( new ShardId("index", "_na_", 0), randomAlphaOfLength(10), primary, ShardRoutingState.INITIALIZING, recoverySource); @@ -244,7 +244,7 @@ protected IndexShard newShard( protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperationListener... listeners) throws IOException { ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, randomAlphaOfLength(5), primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); return newShard(shardRouting, Settings.EMPTY, new InternalEngineFactory(), listeners); } @@ -272,7 +272,7 @@ protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, I protected IndexShard newShard(ShardId shardId, boolean primary, String nodeId, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper searcherWrapper, Runnable globalCheckpointSyncer) throws IOException { ShardRouting shardRouting = TestShardRouting.newShardRouting(shardId, nodeId, primary, ShardRoutingState.INITIALIZING, - primary ? RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); + primary ? RecoverySource.EmptyStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE); return newShard(shardRouting, indexMetaData, searcherWrapper, new InternalEngineFactory(), globalCheckpointSyncer); } @@ -371,7 +371,7 @@ protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMe protected IndexShard reinitShard(IndexShard current, IndexingOperationListener... listeners) throws IOException { final ShardRouting shardRouting = current.routingEntry(); return reinitShard(current, ShardRoutingHelper.initWithSameId(shardRouting, - shardRouting.primary() ? RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE + shardRouting.primary() ? RecoverySource.ExistingStoreRecoverySource.INSTANCE : RecoverySource.PeerRecoverySource.INSTANCE ), listeners); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 5bf8fb6956bfe..bef7705e83533 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -532,7 +532,7 @@ public void testVerifyIndicesPrimaryShardsAreActive() { } else { Index index = new Index(indexToRemove, "_uuid"); ShardId shardId = new ShardId(index, 0); - ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); shardRouting = shardRouting.initialize("node_id", null, 0L); routingTable.add(IndexRoutingTable.builder(index) @@ -656,7 +656,7 @@ private void addJobAndIndices(MetaData.Builder metaData, RoutingTable.Builder ro metaData.put(indexMetaData); Index index = new Index(indexName, "_uuid"); ShardId shardId = new ShardId(index, 0); - ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); shardRouting = shardRouting.initialize("node_id", null, 0L); shardRouting = shardRouting.moveToStarted(); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 3a6082c6cf057..4b8ad1d08aed3 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -349,7 +349,7 @@ private static RoutingTable generateRoutingTable(IndexMetaData indexMetaData, Li true, ShardRoutingState.RELOCATING); } else { shardRouting = ShardRouting.newUnassigned(shardId, true, - RecoverySource.StoreRecoverySource.EMPTY_STORE_INSTANCE, + RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index c7ddb3c4d2427..61224ac0fd735 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.bytes.BytesReference; @@ -60,7 +61,6 @@ import static java.util.Collections.singleton; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; @@ -289,7 +289,8 @@ public void testToXContent() throws IOException { final ShardId shardId = new ShardId("_index", "_index_id", 7); final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message"); - final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, EXISTING_STORE_INSTANCE, unassignedInfo); + final ShardRouting shardRouting = ShardRouting.newUnassigned(shardId, true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, unassignedInfo); final ShardStats mockShardStats = mock(ShardStats.class); when(mockShardStats.getShardRouting()).thenReturn(shardRouting); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 4000969187548..4e5271c520a81 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -55,7 +56,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.contains; @@ -267,8 +267,8 @@ private ClusterState getClusterStateWithSecurityIndex() { } Index index = new Index(securityIndexName, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(Reason.INDEX_CREATED, "")); + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(Reason.INDEX_CREATED, "")); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) .addShard(shardRouting.initialize(randomAlphaOfLength(8), null, shardRouting.getExpectedShardSize()).moveToStarted()) .build(); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index c3a6d7e920d1a..76e84f83137e3 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; @@ -50,7 +51,6 @@ import org.hamcrest.Matchers; import org.junit.Before; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_TEMPLATE_NAME; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN; @@ -106,8 +106,8 @@ public void testIndexWithoutPrimaryShards() throws IOException { final ClusterState.Builder clusterStateBuilder = createClusterState(INDEX_NAME, TEMPLATE_NAME); Index index = new Index(INDEX_NAME, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); String nodeId = ESTestCase.randomAlphaOfLength(8); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) .addShard(shardRouting.initialize(nodeId, null, shardRouting.getExpectedShardSize()) @@ -165,7 +165,8 @@ public void testIndexHealthChangeListeners() throws Exception { clusterStateBuilder.routingTable(RoutingTable.builder() .add(IndexRoutingTable.builder(prevIndex) .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(prevIndex, 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(prevIndex, 0), true, EXISTING_STORE_INSTANCE, + .addShard(ShardRouting.newUnassigned(new ShardId(prevIndex, 0), true, + RecoverySource.ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) .initialize(UUIDs.randomBase64UUID(random()), null, 0L) .moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, ""))) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java index aa4982cce3f84..12474b7a04d59 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/test/SecurityTestUtils.java @@ -5,21 +5,17 @@ */ package org.elasticsearch.xpack.security.test; -import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.Version; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.health.ClusterIndexHealth; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; -import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RecoverySource.ExistingStoreRecoverySource; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -39,9 +35,7 @@ import static java.nio.file.StandardOpenOption.CREATE; import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; import static java.nio.file.StandardOpenOption.WRITE; -import static org.elasticsearch.cluster.routing.RecoverySource.StoreRecoverySource.EXISTING_STORE_INSTANCE; import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_INDEX_NAME; -import static org.junit.Assert.assertEquals; public class SecurityTestUtils { @@ -74,7 +68,7 @@ public static String writeFile(Path folder, String name, String content) { public static RoutingTable buildIndexRoutingTable(String indexName) { Index index = new Index(indexName, UUID.randomUUID().toString()); - ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, EXISTING_STORE_INSTANCE, + ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId(index, 0), true, ExistingStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")); String nodeId = ESTestCase.randomAlphaOfLength(8); IndexShardRoutingTable table = new IndexShardRoutingTable.Builder(new ShardId(index, 0)) @@ -95,60 +89,4 @@ public static MetaData addAliasToMetaData(MetaData metaData, String indexName) { metaDataBuilder.put(IndexMetaData.builder(indexMetaData).putAlias(aliasMetaData)); return metaDataBuilder.build(); } - - public static ClusterIndexHealth getClusterIndexHealth(ClusterHealthStatus status) { - IndexMetaData metaData = IndexMetaData.builder("foo").settings(Settings.builder() - .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) - .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) - .build()) - .build(); - final IndexRoutingTable routingTable; - switch (status) { - case RED: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .build()) - .build(); - break; - case YELLOW: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L)) - .build()) - .build(); - break; - case GREEN: - routingTable = IndexRoutingTable.builder(metaData.getIndex()) - .addIndexShard(new IndexShardRoutingTable.Builder(new ShardId(metaData.getIndex(), 0)) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), true, EXISTING_STORE_INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .addShard(ShardRouting.newUnassigned(new ShardId(metaData.getIndex(), 0), false, - RecoverySource.PeerRecoverySource.INSTANCE, - new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")) - .initialize(ESTestCase.randomAlphaOfLength(8), null, 0L).moveToStarted()) - .build()) - .build(); - break; - default: - throw new IllegalStateException("unknown status: " + status); - } - ClusterIndexHealth health = new ClusterIndexHealth(metaData, routingTable); - assertEquals(status, health.getStatus()); - return health; - } } From 5a38c930fcc42d93475596c92903123e06613bac Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 9 Sep 2018 07:06:55 -0400 Subject: [PATCH 07/12] Add license checks for auto-follow implementation (#33496) This commit adds license checks for the auto-follow implementation. We check the license on put auto-follow patterns, and then for every coordination round we check that the local and remote clusters are licensed for CCR. In the case of non-compliance, we skip coordination yet continue to schedule follow-ups. --- .../elasticsearch/test/MockLogAppender.java | 33 ++++- .../build.gradle | 15 +++ .../xpack/ccr/CcrMultiClusterLicenseIT.java | 44 ++++++- .../java/org/elasticsearch/xpack/ccr/Ccr.java | 2 +- .../xpack/ccr/CcrLicenseChecker.java | 124 ++++++++++++++---- .../ccr/action/AutoFollowCoordinator.java | 76 +++++++---- .../action/CreateAndFollowIndexAction.java | 2 +- .../xpack/ccr/action/FollowIndexAction.java | 2 +- .../TransportPutAutoFollowPatternAction.java | 57 +++++--- .../elasticsearch/xpack/ccr/CcrLicenseIT.java | 68 ++++++++-- ... => NonCompliantLicenseLocalStateCcr.java} | 6 +- .../action/AutoFollowCoordinatorTests.java | 16 +-- 12 files changed, 348 insertions(+), 97 deletions(-) rename x-pack/plugin/ccr/qa/{multi-cluster-with-incompatible-license => multi-cluster-with-non-compliant-license}/build.gradle (61%) rename x-pack/plugin/ccr/qa/{multi-cluster-with-incompatible-license => multi-cluster-with-non-compliant-license}/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java (50%) rename x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/{IncompatibleLicenseLocalStateCcr.java => NonCompliantLicenseLocalStateCcr.java} (80%) diff --git a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java index a53ba046d32c3..895bd7ec77a2b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java +++ b/test/framework/src/main/java/org/elasticsearch/test/MockLogAppender.java @@ -85,7 +85,7 @@ public AbstractEventExpectation(String name, String logger, Level level, String @Override public void match(LogEvent event) { - if (event.getLevel().equals(level) && event.getLoggerName().equals(logger)) { + if (event.getLevel().equals(level) && event.getLoggerName().equals(logger) && innerMatch(event)) { if (Regex.isSimpleMatchPattern(message)) { if (Regex.simpleMatch(message, event.getMessage().getFormattedMessage())) { saw = true; @@ -97,6 +97,11 @@ public void match(LogEvent event) { } } } + + public boolean innerMatch(final LogEvent event) { + return true; + } + } public static class UnseenEventExpectation extends AbstractEventExpectation { @@ -123,6 +128,32 @@ public void assertMatched() { } } + public static class ExceptionSeenEventExpectation extends SeenEventExpectation { + + private final Class clazz; + private final String exceptionMessage; + + public ExceptionSeenEventExpectation( + final String name, + final String logger, + final Level level, + final String message, + final Class clazz, + final String exceptionMessage) { + super(name, logger, level, message); + this.clazz = clazz; + this.exceptionMessage = exceptionMessage; + } + + @Override + public boolean innerMatch(final LogEvent event) { + return event.getThrown() != null + && event.getThrown().getClass() == clazz + && event.getThrown().getMessage().equals(exceptionMessage); + } + + } + public static class PatternSeenEventExcpectation implements LoggingExpectation { protected final String name; diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle similarity index 61% rename from x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle rename to x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle index 1566333e60848..c599903ced12e 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle @@ -20,7 +20,20 @@ leaderClusterTestRunner { systemProperty 'tests.is_leader_cluster', 'true' } +task writeJavaPolicy { + doLast { + final File javaPolicy = file("${buildDir}/tmp/java.policy") + javaPolicy.write( + [ + "grant {", + " permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log\", \"read\";", + "};" + ].join("\n")) + } +} + task followClusterTest(type: RestIntegTestTask) {} +followClusterTest.dependsOn writeJavaPolicy followClusterTestCluster { dependsOn leaderClusterTestRunner @@ -31,8 +44,10 @@ followClusterTestCluster { } followClusterTestRunner { + systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy" systemProperty 'tests.is_leader_cluster', 'false' systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}" + systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}.log" finalizedBy 'leaderClusterTestCluster#stop' } diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java similarity index 50% rename from x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java rename to x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java index 06d9f91c7abb7..c52a4a9b59d78 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-incompatible-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/src/test/java/org/elasticsearch/xpack/ccr/CcrMultiClusterLicenseIT.java @@ -9,11 +9,16 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.test.rest.ESRestTestCase; +import java.nio.file.Files; +import java.util.Iterator; +import java.util.List; import java.util.Locale; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; public class CcrMultiClusterLicenseIT extends ESRestTestCase { @@ -29,7 +34,7 @@ public void testFollowIndex() { if (runningAgainstLeaderCluster == false) { final Request request = new Request("POST", "/follower/_ccr/follow"); request.setJsonEntity("{\"leader_index\": \"leader_cluster:leader\"}"); - assertLicenseIncompatible(request); + assertNonCompliantLicense(request); } } @@ -37,11 +42,44 @@ public void testCreateAndFollowIndex() { if (runningAgainstLeaderCluster == false) { final Request request = new Request("POST", "/follower/_ccr/create_and_follow"); request.setJsonEntity("{\"leader_index\": \"leader_cluster:leader\"}"); - assertLicenseIncompatible(request); + assertNonCompliantLicense(request); } } - private static void assertLicenseIncompatible(final Request request) { + public void testAutoFollow() throws Exception { + if (runningAgainstLeaderCluster == false) { + final Request request = new Request("PUT", "/_ccr/_auto_follow/leader_cluster"); + request.setJsonEntity("{\"leader_index_patterns\":[\"*\"]}"); + client().performRequest(request); + + // parse the logs and ensure that the auto-coordinator skipped coordination on the leader cluster + assertBusy(() -> { + final List lines = Files.readAllLines(PathUtils.get(System.getProperty("log"))); + + final Iterator it = lines.iterator(); + + boolean warn = false; + while (it.hasNext()) { + final String line = it.next(); + if (line.matches(".*\\[WARN\\s*\\]\\[o\\.e\\.x\\.c\\.a\\.AutoFollowCoordinator\\s*\\] \\[node-0\\] " + + "failure occurred during auto-follower coordination")) { + warn = true; + break; + } + } + assertTrue(warn); + assertTrue(it.hasNext()); + final String lineAfterWarn = it.next(); + assertThat( + lineAfterWarn, + equalTo("org.elasticsearch.ElasticsearchStatusException: " + + "can not fetch remote cluster state as the remote cluster [leader_cluster] is not licensed for [ccr]; " + + "the license mode [BASIC] on cluster [leader_cluster] does not enable [ccr]")); + }); + } + } + + private static void assertNonCompliantLicense(final Request request) { final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request)); final String expected = String.format( Locale.ROOT, diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java index cd0561b1c0c60..353a66db26339 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java @@ -126,7 +126,7 @@ public Collection createComponents( return Arrays.asList( ccrLicenseChecker, - new AutoFollowCoordinator(settings, client, threadPool, clusterService) + new AutoFollowCoordinator(settings, client, threadPool, clusterService, ccrLicenseChecker) ); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java index cefa490f4f7e2..f9a5d8fe83035 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrLicenseChecker.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.function.BooleanSupplier; import java.util.function.Consumer; +import java.util.function.Function; /** * Encapsulates licensing checking for CCR. @@ -58,14 +59,13 @@ public boolean isCcrAllowed() { /** * Fetches the leader index metadata from the remote cluster. Before fetching the index metadata, the remote cluster is checked for - * license compatibility with CCR. If the remote cluster is not licensed for CCR, the {@link ActionListener#onFailure(Exception)} method - * of the specified listener is invoked. Otherwise, the specified consumer is invoked with the leader index metadata fetched from the - * remote cluster. + * license compatibility with CCR. If the remote cluster is not licensed for CCR, the {@code onFailure} consumer is is invoked. + * Otherwise, the specified consumer is invoked with the leader index metadata fetched from the remote cluster. * * @param client the client * @param clusterAlias the remote cluster alias * @param leaderIndex the name of the leader index - * @param listener the listener + * @param onFailure the failure consumer * @param leaderIndexMetadataConsumer the leader index metadata consumer * @param the type of response the listener is waiting for */ @@ -73,8 +73,75 @@ public void checkRemoteClusterLicenseAndFetchLeaderIndexMetadata( final Client client, final String clusterAlias, final String leaderIndex, - final ActionListener listener, + final Consumer onFailure, final Consumer leaderIndexMetadataConsumer) { + + final ClusterStateRequest request = new ClusterStateRequest(); + request.clear(); + request.metaData(true); + request.indices(leaderIndex); + checkRemoteClusterLicenseAndFetchClusterState( + client, + clusterAlias, + request, + onFailure, + leaderClusterState -> leaderIndexMetadataConsumer.accept(leaderClusterState.getMetaData().index(leaderIndex)), + licenseCheck -> indexMetadataNonCompliantRemoteLicense(leaderIndex, licenseCheck), + e -> indexMetadataUnknownRemoteLicense(leaderIndex, clusterAlias, e)); + } + + /** + * Fetches the leader cluster state from the remote cluster by the specified cluster state request. Before fetching the cluster state, + * the remote cluster is checked for license compliance with CCR. If the remote cluster is not licensed for CCR, + * the {@code onFailure} consumer is invoked. Otherwise, the specified consumer is invoked with the leader cluster state fetched from + * the remote cluster. + * + * @param client the client + * @param clusterAlias the remote cluster alias + * @param request the cluster state request + * @param onFailure the failure consumer + * @param leaderClusterStateConsumer the leader cluster state consumer + * @param the type of response the listener is waiting for + */ + public void checkRemoteClusterLicenseAndFetchClusterState( + final Client client, + final String clusterAlias, + final ClusterStateRequest request, + final Consumer onFailure, + final Consumer leaderClusterStateConsumer) { + checkRemoteClusterLicenseAndFetchClusterState( + client, + clusterAlias, + request, + onFailure, + leaderClusterStateConsumer, + CcrLicenseChecker::clusterStateNonCompliantRemoteLicense, + e -> clusterStateUnknownRemoteLicense(clusterAlias, e)); + } + + /** + * Fetches the leader cluster state from the remote cluster by the specified cluster state request. Before fetching the cluster state, + * the remote cluster is checked for license compliance with CCR. If the remote cluster is not licensed for CCR, + * the {@code onFailure} consumer is invoked. Otherwise, the specified consumer is invoked with the leader cluster state fetched from + * the remote cluster. + * + * @param client the client + * @param clusterAlias the remote cluster alias + * @param request the cluster state request + * @param onFailure the failure consumer + * @param leaderClusterStateConsumer the leader cluster state consumer + * @param nonCompliantLicense the supplier for when the license state of the remote cluster is non-compliant + * @param unknownLicense the supplier for when the license state of the remote cluster is unknown due to failure + * @param the type of response the listener is waiting for + */ + private void checkRemoteClusterLicenseAndFetchClusterState( + final Client client, + final String clusterAlias, + final ClusterStateRequest request, + final Consumer onFailure, + final Consumer leaderClusterStateConsumer, + final Function nonCompliantLicense, + final Function unknownLicense) { // we have to check the license on the remote cluster new RemoteClusterLicenseChecker(client, XPackLicenseState::isCcrAllowedForOperationMode).checkRemoteClusterLicenses( Collections.singletonList(clusterAlias), @@ -83,35 +150,25 @@ public void checkRemoteClusterLicenseAndFetchLeaderIndexMetadata( @Override public void onResponse(final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { if (licenseCheck.isSuccess()) { - final Client remoteClient = client.getRemoteClusterClient(clusterAlias); - final ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.clear(); - clusterStateRequest.metaData(true); - clusterStateRequest.indices(leaderIndex); - final ActionListener clusterStateListener = ActionListener.wrap( - r -> { - final ClusterState remoteClusterState = r.getState(); - final IndexMetaData leaderIndexMetadata = - remoteClusterState.getMetaData().index(leaderIndex); - leaderIndexMetadataConsumer.accept(leaderIndexMetadata); - }, - listener::onFailure); + final Client leaderClient = client.getRemoteClusterClient(clusterAlias); + final ActionListener clusterStateListener = + ActionListener.wrap(s -> leaderClusterStateConsumer.accept(s.getState()), onFailure); // following an index in remote cluster, so use remote client to fetch leader index metadata - remoteClient.admin().cluster().state(clusterStateRequest, clusterStateListener); + leaderClient.admin().cluster().state(request, clusterStateListener); } else { - listener.onFailure(incompatibleRemoteLicense(leaderIndex, licenseCheck)); + onFailure.accept(nonCompliantLicense.apply(licenseCheck)); } } @Override public void onFailure(final Exception e) { - listener.onFailure(unknownRemoteLicense(leaderIndex, clusterAlias, e)); + onFailure.accept(unknownLicense.apply(e)); } }); } - private static ElasticsearchStatusException incompatibleRemoteLicense( + private static ElasticsearchStatusException indexMetadataNonCompliantRemoteLicense( final String leaderIndex, final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { final String clusterAlias = licenseCheck.remoteClusterLicenseInfo().clusterAlias(); final String message = String.format( @@ -127,7 +184,21 @@ private static ElasticsearchStatusException incompatibleRemoteLicense( return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST); } - private static ElasticsearchStatusException unknownRemoteLicense( + private static ElasticsearchStatusException clusterStateNonCompliantRemoteLicense( + final RemoteClusterLicenseChecker.LicenseCheck licenseCheck) { + final String clusterAlias = licenseCheck.remoteClusterLicenseInfo().clusterAlias(); + final String message = String.format( + Locale.ROOT, + "can not fetch remote cluster state as the remote cluster [%s] is not licensed for [ccr]; %s", + clusterAlias, + RemoteClusterLicenseChecker.buildErrorMessage( + "ccr", + licenseCheck.remoteClusterLicenseInfo(), + RemoteClusterLicenseChecker::isLicensePlatinumOrTrial)); + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST); + } + + private static ElasticsearchStatusException indexMetadataUnknownRemoteLicense( final String leaderIndex, final String clusterAlias, final Exception cause) { final String message = String.format( Locale.ROOT, @@ -138,4 +209,11 @@ private static ElasticsearchStatusException unknownRemoteLicense( return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST, cause); } + private static ElasticsearchStatusException clusterStateUnknownRemoteLicense(final String clusterAlias, final Exception cause) { + final String message = String.format( + Locale.ROOT, + "can not fetch remote cluster state as the license state of the remote cluster [%s] could not be determined", clusterAlias); + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST, cause); + } + } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java index 234fe32cdd0ee..639cd4d5782ab 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java @@ -21,7 +21,9 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.index.Index; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.ccr.CcrSettings; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -30,6 +32,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -47,22 +50,32 @@ public class AutoFollowCoordinator implements ClusterStateApplier { private final TimeValue pollInterval; private final ThreadPool threadPool; private final ClusterService clusterService; + private final CcrLicenseChecker ccrLicenseChecker; private volatile boolean localNodeMaster = false; - public AutoFollowCoordinator(Settings settings, - Client client, - ThreadPool threadPool, - ClusterService clusterService) { + public AutoFollowCoordinator( + Settings settings, + Client client, + ThreadPool threadPool, + ClusterService clusterService, + CcrLicenseChecker ccrLicenseChecker) { this.client = client; this.threadPool = threadPool; this.clusterService = clusterService; + this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker"); this.pollInterval = CcrSettings.CCR_AUTO_FOLLOW_POLL_INTERVAL.get(settings); clusterService.addStateApplier(this); } private void doAutoFollow() { + if (ccrLicenseChecker.isCcrAllowed() == false) { + // TODO: set non-compliant status on auto-follow coordination that can be viewed via a stats API + LOGGER.warn("skipping auto-follower coordination", LicenseUtils.newComplianceException("ccr")); + threadPool.schedule(pollInterval, ThreadPool.Names.SAME, this::doAutoFollow); + return; + } if (localNodeMaster == false) { return; } @@ -80,23 +93,32 @@ private void doAutoFollow() { Consumer handler = e -> { if (e != null) { - LOGGER.warn("Failure occurred during auto following indices", e); + LOGGER.warn("failure occurred during auto-follower coordination", e); } threadPool.schedule(pollInterval, ThreadPool.Names.SAME, this::doAutoFollow); }; - AutoFollower operation = new AutoFollower(client, handler, followerClusterState) { + AutoFollower operation = new AutoFollower(handler, followerClusterState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { - ClusterStateRequest request = new ClusterStateRequest(); + void getLeaderClusterState(final String leaderClusterAlias, final BiConsumer handler) { + final ClusterStateRequest request = new ClusterStateRequest(); request.clear(); request.metaData(true); - leaderClient.admin().cluster().state(request, - ActionListener.wrap( - r -> handler.accept(r.getState(), null), - e -> handler.accept(null, e) - ) - ); + + if ("_local_".equals(leaderClusterAlias)) { + client.admin().cluster().state( + request, ActionListener.wrap(r -> handler.accept(r.getState(), null), e -> handler.accept(null, e))); + } else { + final Client leaderClient = client.getRemoteClusterClient(leaderClusterAlias); + // TODO: set non-compliant status on auto-follow coordination that can be viewed via a stats API + ccrLicenseChecker.checkRemoteClusterLicenseAndFetchClusterState( + leaderClient, + leaderClusterAlias, + request, + e -> handler.accept(null, e), + leaderClusterState -> handler.accept(leaderClusterState, null)); + } + } @Override @@ -143,7 +165,6 @@ public void applyClusterState(ClusterChangedEvent event) { abstract static class AutoFollower { - private final Client client; private final Consumer handler; private final ClusterState followerClusterState; private final AutoFollowMetadata autoFollowMetadata; @@ -151,8 +172,7 @@ abstract static class AutoFollower { private final CountDown autoFollowPatternsCountDown; private final AtomicReference autoFollowPatternsErrorHolder = new AtomicReference<>(); - AutoFollower(Client client, Consumer handler, ClusterState followerClusterState) { - this.client = client; + AutoFollower(final Consumer handler, final ClusterState followerClusterState) { this.handler = handler; this.followerClusterState = followerClusterState; this.autoFollowMetadata = followerClusterState.getMetaData().custom(AutoFollowMetadata.TYPE); @@ -163,10 +183,9 @@ void autoFollowIndices() { for (Map.Entry entry : autoFollowMetadata.getPatterns().entrySet()) { String clusterAlias = entry.getKey(); AutoFollowPattern autoFollowPattern = entry.getValue(); - Client leaderClient = clusterAlias.equals("_local_") ? client : client.getRemoteClusterClient(clusterAlias); List followedIndices = autoFollowMetadata.getFollowedLeaderIndexUUIDs().get(clusterAlias); - getLeaderClusterState(leaderClient, (leaderClusterState, e) -> { + getLeaderClusterState(clusterAlias, (leaderClusterState, e) -> { if (leaderClusterState != null) { assert e == null; handleClusterAlias(clusterAlias, autoFollowPattern, followedIndices, leaderClusterState); @@ -289,18 +308,17 @@ static Function recordLeaderIndexAsFollowFunction(St }; } - // abstract methods to make unit testing possible: - - abstract void getLeaderClusterState(Client leaderClient, - BiConsumer handler); + /** + * Fetch the cluster state from the leader with the specified cluster alias + * + * @param leaderClusterAlias the cluster alias of the leader + * @param handler the callback to invoke + */ + abstract void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler); - abstract void createAndFollow(FollowIndexAction.Request followRequest, - Runnable successHandler, - Consumer failureHandler); + abstract void createAndFollow(FollowIndexAction.Request followRequest, Runnable successHandler, Consumer failureHandler); - abstract void updateAutoFollowMetadata(Function updateFunction, - Consumer handler); + abstract void updateAutoFollowMetadata(Function updateFunction, Consumer handler); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java index 2e36bca293225..cf77bf8112f91 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java @@ -255,7 +255,7 @@ private void createFollowerIndexAndFollowRemoteIndex( client, clusterAlias, leaderIndex, - listener, + listener::onFailure, leaderIndexMetaData -> createFollowerIndex(leaderIndexMetaData, request, listener)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java index 17b7bbe674b38..2a14c4e9a50bb 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java @@ -370,7 +370,7 @@ private void followRemoteIndex( client, clusterAlias, leaderIndex, - listener, + listener::onFailure, leaderIndexMetadata -> { try { start(request, clusterAlias, leaderIndexMetadata, followerIndexMetadata, listener); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java index 3d3e342c0cd3e..a4ff9511cfbd8 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java @@ -21,8 +21,10 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ccr.CcrLicenseChecker; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata.AutoFollowPattern; @@ -30,20 +32,29 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; public class TransportPutAutoFollowPatternAction extends TransportMasterNodeAction { private final Client client; + private final CcrLicenseChecker ccrLicenseChecker; @Inject - public TransportPutAutoFollowPatternAction(Settings settings, TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, ActionFilters actionFilters, Client client, - IndexNameExpressionResolver indexNameExpressionResolver) { + public TransportPutAutoFollowPatternAction( + final Settings settings, + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final Client client, + final IndexNameExpressionResolver indexNameExpressionResolver, + final CcrLicenseChecker ccrLicenseChecker) { super(settings, PutAutoFollowPatternAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, PutAutoFollowPatternAction.Request::new); this.client = client; + this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker, "ccrLicenseChecker"); } @Override @@ -60,6 +71,10 @@ protected AcknowledgedResponse newResponse() { protected void masterOperation(PutAutoFollowPatternAction.Request request, ClusterState state, ActionListener listener) throws Exception { + if (ccrLicenseChecker.isCcrAllowed() == false) { + listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } final Client leaderClient; if (request.getLeaderClusterAlias().equals("_local_")) { leaderClient = client; @@ -71,22 +86,26 @@ protected void masterOperation(PutAutoFollowPatternAction.Request request, clusterStateRequest.clear(); clusterStateRequest.metaData(true); - leaderClient.admin().cluster().state(clusterStateRequest, ActionListener.wrap(clusterStateResponse -> { - final ClusterState leaderClusterState = clusterStateResponse.getState(); - clusterService.submitStateUpdateTask("put-auto-follow-pattern-" + request.getLeaderClusterAlias(), - new AckedClusterStateUpdateTask(request, listener) { - - @Override - protected AcknowledgedResponse newResponse(boolean acknowledged) { - return new AcknowledgedResponse(acknowledged); - } - - @Override - public ClusterState execute(ClusterState currentState) throws Exception { - return innerPut(request, currentState, leaderClusterState); - } - }); - }, listener::onFailure)); + leaderClient.admin().cluster().state( + clusterStateRequest, + ActionListener.wrap( + clusterStateResponse -> { + final ClusterState leaderClusterState = clusterStateResponse.getState(); + clusterService.submitStateUpdateTask("put-auto-follow-pattern-" + request.getLeaderClusterAlias(), + new AckedClusterStateUpdateTask(request, listener) { + + @Override + protected AcknowledgedResponse newResponse(boolean acknowledged) { + return new AcknowledgedResponse(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return innerPut(request, currentState, leaderClusterState); + } + }); + }, + listener::onFailure)); } static ClusterState innerPut(PutAutoFollowPatternAction.Request request, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index 675758903bf27..05383b280e6ce 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -6,15 +6,22 @@ package org.elasticsearch.xpack.ccr; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.MockLogAppender; +import org.elasticsearch.xpack.ccr.action.AutoFollowCoordinator; import org.elasticsearch.xpack.ccr.action.CcrStatsAction; import org.elasticsearch.xpack.ccr.action.CreateAndFollowIndexAction; import org.elasticsearch.xpack.ccr.action.FollowIndexAction; +import org.elasticsearch.xpack.ccr.action.PutAutoFollowPatternAction; import org.elasticsearch.xpack.ccr.action.ShardFollowNodeTask; import java.util.Collection; @@ -28,10 +35,10 @@ public class CcrLicenseIT extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return Collections.singletonList(IncompatibleLicenseLocalStateCcr.class); + return Collections.singletonList(NonCompliantLicenseLocalStateCcr.class); } - public void testThatFollowingIndexIsUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatFollowingIndexIsUnavailableWithNonCompliantLicense() throws InterruptedException { final FollowIndexAction.Request followRequest = getFollowRequest(); final CountDownLatch latch = new CountDownLatch(1); client().execute( @@ -45,14 +52,14 @@ public void onResponse(final AcknowledgedResponse response) { @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); latch.await(); } - public void testThatCreateAndFollowingIndexIsUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatCreateAndFollowingIndexIsUnavailableWithNonCompliantLicense() throws InterruptedException { final FollowIndexAction.Request followRequest = getFollowRequest(); final CreateAndFollowIndexAction.Request createAndFollowRequest = new CreateAndFollowIndexAction.Request(followRequest); final CountDownLatch latch = new CountDownLatch(1); @@ -67,14 +74,14 @@ public void onResponse(final CreateAndFollowIndexAction.Response response) { @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); latch.await(); } - public void testThatCcrStatsAreUnavailableWithIncompatibleLicense() throws InterruptedException { + public void testThatCcrStatsAreUnavailableWithNonCompliantLicense() throws InterruptedException { final CountDownLatch latch = new CountDownLatch(1); client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.TasksRequest(), new ActionListener() { @Override @@ -84,7 +91,7 @@ public void onResponse(final CcrStatsAction.TasksResponse tasksResponse) { @Override public void onFailure(final Exception e) { - assertIncompatibleLicense(e); + assertNonCompliantLicense(e); latch.countDown(); } }); @@ -92,7 +99,52 @@ public void onFailure(final Exception e) { latch.await(); } - private void assertIncompatibleLicense(final Exception e) { + public void testThatPutAutoFollowPatternsIsUnavailableWithNonCompliantLicense() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + final PutAutoFollowPatternAction.Request request = new PutAutoFollowPatternAction.Request(); + request.setLeaderClusterAlias("leader"); + request.setLeaderIndexPatterns(Collections.singletonList("*")); + client().execute( + PutAutoFollowPatternAction.INSTANCE, + request, + new ActionListener() { + @Override + public void onResponse(final AcknowledgedResponse response) { + latch.countDown(); + fail(); + } + + @Override + public void onFailure(final Exception e) { + assertNonCompliantLicense(e); + latch.countDown(); + } + }); + latch.await(); + } + + public void testAutoFollowCoordinatorLogsSkippingAutoFollowCoordinationWithNonCompliantLicense() throws Exception { + final Logger logger = LogManager.getLogger(AutoFollowCoordinator.class); + final MockLogAppender appender = new MockLogAppender(); + appender.start(); + appender.addExpectation( + new MockLogAppender.ExceptionSeenEventExpectation( + getTestName(), + logger.getName(), + Level.WARN, + "skipping auto-follower coordination", + ElasticsearchSecurityException.class, + "current license is non-compliant for [ccr]")); + Loggers.addAppender(logger, appender); + try { + assertBusy(appender::assertAllExpectationsMatched); + } finally { + Loggers.removeAppender(logger, appender); + appender.stop(); + } + } + + private void assertNonCompliantLicense(final Exception e) { assertThat(e, instanceOf(ElasticsearchSecurityException.class)); assertThat(e.getMessage(), equalTo("current license is non-compliant for [ccr]")); } diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java similarity index 80% rename from x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java rename to x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java index c4b765d3c65ea..f960668a7dff1 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IncompatibleLicenseLocalStateCcr.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/NonCompliantLicenseLocalStateCcr.java @@ -12,16 +12,16 @@ import java.nio.file.Path; -public class IncompatibleLicenseLocalStateCcr extends LocalStateCompositeXPackPlugin { +public class NonCompliantLicenseLocalStateCcr extends LocalStateCompositeXPackPlugin { - public IncompatibleLicenseLocalStateCcr(final Settings settings, final Path configPath) throws Exception { + public NonCompliantLicenseLocalStateCcr(final Settings settings, final Path configPath) throws Exception { super(settings, configPath); plugins.add(new Ccr(settings, new CcrLicenseChecker(() -> false)) { @Override protected XPackLicenseState getLicenseState() { - return IncompatibleLicenseLocalStateCcr.this.getLicenseState(); + return NonCompliantLicenseLocalStateCcr.this.getLicenseState(); } }); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java index dd1376a4d7a73..2ef841292322a 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java @@ -66,9 +66,9 @@ public void testAutoFollower() { invoked[0] = true; assertThat(e, nullValue()); }; - AutoFollower autoFollower = new AutoFollower(client, handler, currentState) { + AutoFollower autoFollower = new AutoFollower(handler, currentState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } @@ -113,9 +113,9 @@ public void testAutoFollowerClusterStateApiFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(null, failure); } @@ -161,9 +161,9 @@ public void testAutoFollowerUpdateClusterStateFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } @@ -211,9 +211,9 @@ public void testAutoFollowerCreateAndFollowApiCallFailure() { invoked[0] = true; assertThat(e, sameInstance(failure)); }; - AutoFollower autoFollower = new AutoFollower(client, handler, followerState) { + AutoFollower autoFollower = new AutoFollower(handler, followerState) { @Override - void getLeaderClusterState(Client leaderClient, BiConsumer handler) { + void getLeaderClusterState(String leaderClusterAlias, BiConsumer handler) { handler.accept(leaderState, null); } From c67b0ba33edd361b49e2915d04ecf74e7e81f4d5 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 9 Sep 2018 07:16:56 -0400 Subject: [PATCH 08/12] Create temporary directory if needed in CCR test In the multi-cluster-with-non-compliant-license tests, we try to write out a java.policy to a temporary directory. However, if this temporary directory does not already exist then writing the java.policy file will fail. This commit ensures that the temporary directory exists before we attempt to write the java.policy file. --- .../multi-cluster-with-non-compliant-license/build.gradle | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle index c599903ced12e..845c9df533dba 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle +++ b/x-pack/plugin/ccr/qa/multi-cluster-with-non-compliant-license/build.gradle @@ -22,7 +22,11 @@ leaderClusterTestRunner { task writeJavaPolicy { doLast { - final File javaPolicy = file("${buildDir}/tmp/java.policy") + final File tmp = file("${buildDir}/tmp") + if (tmp.exists() == false && tmp.mkdirs() == false) { + throw new GradleException("failed to create temporary directory [${tmp}]") + } + final File javaPolicy = file("${tmp}/java.policy") javaPolicy.write( [ "grant {", From 9073dbefd6d5a1441ac11e7990e181fa74f3ce15 Mon Sep 17 00:00:00 2001 From: "S.Y. Wang" Date: Sun, 9 Sep 2018 20:47:47 +0900 Subject: [PATCH 09/12] HLRC: Add put stored script support to high-level rest client (#31323) Relates to #27205 --- .../client/RequestConverters.java | 14 ++ .../client/RestHighLevelClient.java | 64 +++++--- .../client/RequestConvertersTests.java | 37 +++++ .../client/RestHighLevelClientTests.java | 1 - .../elasticsearch/client/StoredScriptsIT.java | 58 ++++---- .../StoredScriptsDocumentationIT.java | 139 ++++++++++++++++-- .../high-level/script/put_script.asciidoc | 106 +++++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../storedscripts/PutStoredScriptRequest.java | 12 +- .../PutStoredScriptRequestTests.java | 29 ++++ 10 files changed, 403 insertions(+), 59 deletions(-) create mode 100644 docs/java-rest/high-level/script/put_script.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 7fe3e08f3afb0..89f81512bc9d2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -32,6 +32,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.analyze.AnalyzeRequest; @@ -887,6 +888,19 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) t return request; } + static Request putScript(PutStoredScriptRequest putStoredScriptRequest) throws IOException { + String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(putStoredScriptRequest.id()).build(); + Request request = new Request(HttpPost.METHOD_NAME, endpoint); + Params params = new Params(request); + params.withTimeout(putStoredScriptRequest.timeout()); + params.withMasterTimeout(putStoredScriptRequest.masterNodeTimeout()); + if (Strings.hasText(putStoredScriptRequest.context())) { + params.putParam("context", putStoredScriptRequest.context()); + } + request.setEntity(createEntity(putStoredScriptRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request analyze(AnalyzeRequest request) throws IOException { EndpointBuilder builder = new EndpointBuilder(); String index = request.index(); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 17f8f65943012..687290abe8866 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; @@ -121,36 +122,36 @@ import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedAvg; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedGeoBounds; import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedGeoCentroid; -import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedMax; -import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedMin; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.elasticsearch.search.aggregations.metrics.InternalHDRPercentiles; -import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentileRanks; -import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentiles; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.InternalTDigestPercentiles; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.ParsedAvg; +import org.elasticsearch.search.aggregations.metrics.ParsedCardinality; +import org.elasticsearch.search.aggregations.metrics.ParsedExtendedStats; +import org.elasticsearch.search.aggregations.metrics.ParsedGeoBounds; +import org.elasticsearch.search.aggregations.metrics.ParsedGeoCentroid; +import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentileRanks; +import org.elasticsearch.search.aggregations.metrics.ParsedHDRPercentiles; +import org.elasticsearch.search.aggregations.metrics.ParsedMax; +import org.elasticsearch.search.aggregations.metrics.ParsedMin; +import org.elasticsearch.search.aggregations.metrics.ParsedScriptedMetric; +import org.elasticsearch.search.aggregations.metrics.ParsedStats; +import org.elasticsearch.search.aggregations.metrics.ParsedSum; import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentileRanks; import org.elasticsearch.search.aggregations.metrics.ParsedTDigestPercentiles; -import org.elasticsearch.search.aggregations.metrics.ParsedScriptedMetric; +import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; +import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; import org.elasticsearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedStats; import org.elasticsearch.search.aggregations.metrics.StatsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedExtendedStats; -import org.elasticsearch.search.aggregations.metrics.ParsedSum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedTopHits; import org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.ParsedValueCount; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.ParsedSimpleValue; @@ -1050,6 +1051,35 @@ public void deleteScriptAsync(DeleteStoredScriptRequest request, RequestOptions AcknowledgedResponse::fromXContent, listener, emptySet()); } + /** + * Puts an stored script using the Scripting API. + * See Scripting API + * on elastic.co + * @param putStoredScriptRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public AcknowledgedResponse putScript(PutStoredScriptRequest putStoredScriptRequest, + RequestOptions options) throws IOException { + return performRequestAndParseEntity(putStoredScriptRequest, RequestConverters::putScript, options, + AcknowledgedResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously puts an stored script using the Scripting API. + * See Scripting API + * on elastic.co + * @param putStoredScriptRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void putScriptAsync(PutStoredScriptRequest putStoredScriptRequest, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(putStoredScriptRequest, RequestConverters::putScript, options, + AcknowledgedResponse::fromXContent, listener, emptySet()); + } + /** * Asynchronously executes a request using the Field Capabilities API. * See Field Capabilities API diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 840df49b47811..6f48d305a7799 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; @@ -1991,6 +1992,42 @@ public void testGetTemplateRequest() throws Exception { assertThat(request.getEntity(), nullValue()); } + public void testPutScript() throws Exception { + PutStoredScriptRequest putStoredScriptRequest = new PutStoredScriptRequest(); + + String id = randomAlphaOfLengthBetween(5, 10); + putStoredScriptRequest.id(id); + + XContentType xContentType = randomFrom(XContentType.values()); + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + builder.startObject(); + builder.startObject("script") + .field("lang", "painless") + .field("source", "Math.log(_score * 2) + params.multiplier") + .endObject(); + builder.endObject(); + + putStoredScriptRequest.content(BytesReference.bytes(builder), xContentType); + } + + Map expectedParams = new HashMap<>(); + setRandomMasterTimeout(putStoredScriptRequest, expectedParams); + setRandomTimeout(putStoredScriptRequest::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + + if (randomBoolean()) { + String context = randomAlphaOfLengthBetween(5, 10); + putStoredScriptRequest.context(context); + expectedParams.put("context", context); + } + + Request request = RequestConverters.putScript(putStoredScriptRequest); + + assertThat(request.getEndpoint(), equalTo("/_scripts/" + id)); + assertThat(request.getParameters(), equalTo(expectedParams)); + assertNotNull(request.getEntity()); + assertToXContentBody(putStoredScriptRequest, request.getEntity()); + } + public void testAnalyzeRequest() throws Exception { AnalyzeRequest indexAnalyzeRequest = new AnalyzeRequest() .text("Here is some text") diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index b6562cd44cd55..3bd47306e5e10 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -658,7 +658,6 @@ public void testApiNamingConventions() throws Exception { "indices.get_upgrade", "indices.put_alias", "mtermvectors", - "put_script", "reindex_rethrottle", "render_search_template", "scripts_painless_execute", diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java index 1d693eee8396e..b15467d24ba2b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/StoredScriptsIT.java @@ -1,4 +1,5 @@ -package org.elasticsearch.client;/* +package org.elasticsearch.client; +/* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright @@ -17,27 +18,27 @@ * under the License. */ -import org.apache.http.util.EntityUtils; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.Script; import org.elasticsearch.script.StoredScriptSource; import java.util.Collections; +import java.util.Map; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; public class StoredScriptsIT extends ESRestHighLevelClientTestCase { - final String id = "calculate-score"; + private static final String id = "calculate-score"; public void testGetStoredScript() throws Exception { final StoredScriptSource scriptSource = @@ -45,13 +46,9 @@ public void testGetStoredScript() throws Exception { "Math.log(_score * 2) + params.my_modifier", Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request putRequest = new Request("PUT", "/_scripts/calculate-score"); - putRequest.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(putRequest); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); GetStoredScriptRequest getRequest = new GetStoredScriptRequest("calculate-score"); getRequest.masterNodeTimeout("50s"); @@ -68,22 +65,14 @@ public void testDeleteStoredScript() throws Exception { "Math.log(_score * 2) + params.my_modifier", Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request putRequest = new Request("PUT", "/_scripts/" + id); - putRequest.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(putRequest); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); DeleteStoredScriptRequest deleteRequest = new DeleteStoredScriptRequest(id); deleteRequest.masterNodeTimeout("50s"); deleteRequest.timeout("50s"); - - AcknowledgedResponse deleteResponse = execute(deleteRequest, highLevelClient()::deleteScript, - highLevelClient()::deleteScriptAsync); - - assertThat(deleteResponse.isAcknowledged(), equalTo(true)); + assertAcked(execute(deleteRequest, highLevelClient()::deleteScript, highLevelClient()::deleteScriptAsync)); GetStoredScriptRequest getRequest = new GetStoredScriptRequest(id); @@ -92,4 +81,21 @@ public void testDeleteStoredScript() throws Exception { highLevelClient()::getScriptAsync)); assertThat(statusException.status(), equalTo(RestStatus.NOT_FOUND)); } + + public void testPutScript() throws Exception { + final StoredScriptSource scriptSource = + new StoredScriptSource("painless", + "Math.log(_score * 2) + params.my_modifier", + Collections.singletonMap(Script.CONTENT_TYPE_OPTION, XContentType.JSON.mediaType())); + + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); + + Map script = getAsMap("/_scripts/" + id); + assertThat(extractValue("_id", script), equalTo(id)); + assertThat(extractValue("found", script), equalTo(true)); + assertThat(extractValue("script.lang", script), equalTo("painless")); + assertThat(extractValue("script.source", script), equalTo("Math.log(_score * 2) + params.my_modifier")); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java index fc38090ef5b5b..c5d53abd978e1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/StoredScriptsDocumentationIT.java @@ -17,21 +17,21 @@ * under the License. */ -import org.apache.http.util.EntityUtils; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse; +import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; -import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.Response; import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.Script; import org.elasticsearch.script.StoredScriptSource; @@ -42,7 +42,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; /** @@ -187,14 +188,124 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } + public void testPutScript() throws Exception { + RestHighLevelClient client = highLevelClient(); + + { + // tag::put-stored-script-request + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); // <1> + request.content(new BytesArray( + "{\n" + + "\"script\": {\n" + + "\"lang\": \"painless\",\n" + + "\"source\": \"Math.log(_score * 2) + params.multiplier\"" + + "}\n" + + "}\n" + ), XContentType.JSON); // <2> + // end::put-stored-script-request + + // tag::put-stored-script-context + request.context("context"); // <1> + // end::put-stored-script-context + + // tag::put-stored-script-timeout + request.timeout(TimeValue.timeValueMinutes(2)); // <1> + request.timeout("2m"); // <2> + // end::put-stored-script-timeout + + // tag::put-stored-script-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> + // end::put-stored-script-masterTimeout + } + + { + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); + + // tag::put-stored-script-content-painless + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("script"); + { + builder.field("lang", "painless"); + builder.field("source", "Math.log(_score * 2) + params.multiplier"); + } + builder.endObject(); + } + builder.endObject(); + request.content(BytesReference.bytes(builder), XContentType.JSON); // <1> + // end::put-stored-script-content-painless + + + // tag::put-stored-script-execute + AcknowledgedResponse putStoredScriptResponse = client.putScript(request, RequestOptions.DEFAULT); + // end::put-stored-script-execute + + // tag::put-stored-script-response + boolean acknowledged = putStoredScriptResponse.isAcknowledged(); // <1> + // end::put-stored-script-response + + assertTrue(acknowledged); + + // tag::put-stored-script-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(AcknowledgedResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::put-stored-script-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::put-stored-script-execute-async + client.putScriptAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::put-stored-script-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + + { + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("id"); + + // tag::put-stored-script-content-mustache + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder.startObject(); + { + builder.startObject("script"); + { + builder.field("lang", "mustache"); + builder.field("source", "{\"query\":{\"match\":{\"title\":\"{{query_string}}\"}}}"); + } + builder.endObject(); + } + builder.endObject(); + request.content(BytesReference.bytes(builder), XContentType.JSON); // <1> + // end::put-stored-script-content-mustache + + client.putScript(request, RequestOptions.DEFAULT); + + Map script = getAsMap("/_scripts/id"); + assertThat(extractValue("script.lang", script), equalTo("mustache")); + assertThat(extractValue("script.source", script), equalTo("{\"query\":{\"match\":{\"title\":\"{{query_string}}\"}}}")); + } + } + private void putStoredScript(String id, StoredScriptSource scriptSource) throws IOException { - final String script = Strings.toString(scriptSource.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS)); - // TODO: change to HighLevel PutStoredScriptRequest when it will be ready - // so far - using low-level REST API - Request request = new Request("PUT", "/_scripts/" + id); - request.setJsonEntity("{\"script\":" + script + "}"); - Response putResponse = adminClient().performRequest(request); - assertEquals(putResponse.getStatusLine().getReasonPhrase(), 200, putResponse.getStatusLine().getStatusCode()); - assertEquals("{\"acknowledged\":true}", EntityUtils.toString(putResponse.getEntity())); + PutStoredScriptRequest request = + new PutStoredScriptRequest(id, "search", new BytesArray("{}"), XContentType.JSON, scriptSource); + assertAcked(execute(request, highLevelClient()::putScript, highLevelClient()::putScriptAsync)); } } diff --git a/docs/java-rest/high-level/script/put_script.asciidoc b/docs/java-rest/high-level/script/put_script.asciidoc new file mode 100644 index 0000000000000..acc80e82d11e6 --- /dev/null +++ b/docs/java-rest/high-level/script/put_script.asciidoc @@ -0,0 +1,106 @@ +[[java-rest-high-put-stored-script]] +=== Put Stored Script API + +[[java-rest-high-put-stored-script-request]] +==== Put Stored Script Request + +A `PutStoredScriptRequest` requires an `id` and `content`: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-request] +-------------------------------------------------- +<1> The id of the script +<2> The content of the script + +[[java-rest-high-put-stored-script-content]] +==== Content +The content of a script can be written in different languages and provided in +different ways: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-content-painless] +-------------------------------------------------- +<1> Specify a painless script and provided as `XContentBuilder` object. +Note that the builder needs to be passed as a `BytesReference` object + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-content-mustache] +-------------------------------------------------- +<1> Specify a mustache script and provided as `XContentBuilder` object. +Note that value of source can be directly provided as a JSON string + +==== Optional arguments +The following arguments can optionally be provided: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-context] +-------------------------------------------------- +<1> The context the script should be executed in. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-timeout] +-------------------------------------------------- +<1> Timeout to wait for the all the nodes to acknowledge the script creation as a `TimeValue` +<2> Timeout to wait for the all the nodes to acknowledge the script creation as a `String` + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue` +<2> Timeout to connect to the master node as a `String` + +[[java-rest-high-put-stored-script-sync]] +==== Synchronous Execution +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute] +-------------------------------------------------- + +[[java-rest-high-put-stored-script-async]] +==== Asynchronous Execution + +The asynchronous execution of a put stored script request requires both the `PutStoredScriptRequest` +instance and an `ActionListener` instance to be passed to the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute-async] +-------------------------------------------------- +<1> The `PutStoredScriptRequest` to execute and the `ActionListener` to use when +the execution completes + +[[java-rest-high-put-stored-script-listener]] +===== Action Listener + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `AcknowledgedResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument + +[[java-rest-high-put-stored-script-response]] +==== Put Stored Script Response + +The returned `AcknowledgedResponse` allows to retrieve information about the +executed operation as follows: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/StoredScriptsDocumentationIT.java[put-stored-script-response] +-------------------------------------------------- +<1> Indicates whether all of the nodes have acknowledged the request \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 8d49724353e6f..8d92653ce5702 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -189,9 +189,11 @@ include::tasks/cancel_tasks.asciidoc[] The Java High Level REST Client supports the following Scripts APIs: * <> +* <> * <> include::script/get_script.asciidoc[] +include::script/put_script.asciidoc[] include::script/delete_script.asciidoc[] == Licensing APIs diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java index d02d6272c9514..e7c5a07f56874 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequest.java @@ -25,6 +25,8 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.StoredScriptSource; @@ -34,7 +36,7 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; -public class PutStoredScriptRequest extends AcknowledgedRequest { +public class PutStoredScriptRequest extends AcknowledgedRequest implements ToXContent { private String id; private String context; @@ -160,4 +162,12 @@ public String toString() { (context != null ? ", context [" + context + "]" : "") + ", content [" + source + "]}"; } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("script"); + source.toXContent(builder, params); + + return builder; + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java index 2ca71fabbc7dc..821c75c2ed7d3 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/storedscripts/PutStoredScriptRequestTests.java @@ -20,8 +20,11 @@ package org.elasticsearch.action.admin.cluster.storedscripts; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.script.StoredScriptSource; import org.elasticsearch.test.ESTestCase; @@ -48,4 +51,30 @@ public void testSerialization() throws IOException { } } } + + public void testToXContent() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = XContentBuilder.builder(xContentType.xContent()); + builder.startObject(); + builder.startObject("script") + .field("lang", "painless") + .field("source", "Math.log(_score * 2) + params.multiplier") + .endObject(); + builder.endObject(); + + BytesReference expectedRequestBody = BytesReference.bytes(builder); + + PutStoredScriptRequest request = new PutStoredScriptRequest(); + request.id("test1"); + request.content(expectedRequestBody, xContentType); + + XContentBuilder requestBuilder = XContentBuilder.builder(xContentType.xContent()); + requestBuilder.startObject(); + request.toXContent(requestBuilder, ToXContent.EMPTY_PARAMS); + requestBuilder.endObject(); + + BytesReference actualRequestBody = BytesReference.bytes(requestBuilder); + + assertEquals(expectedRequestBody, actualRequestBody); + } } From edc492419be3a30550f4b5816d438867d7a7c3c0 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 9 Sep 2018 09:52:40 -0400 Subject: [PATCH 10/12] Add latch countdown on failure in CCR license tests (#33548) We have some listeners in the CCR license tests that invoke Assert#fail if the onSuccess method for the listener is unexpectedly invoked. This can leave the main test thread hanging until the test suite times out rather than failing quickly. This commit adds some latch countdowns so that we fail quickly if these cases are hit. --- .../test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java index 05383b280e6ce..06cafc4777a49 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrLicenseIT.java @@ -47,6 +47,7 @@ public void testThatFollowingIndexIsUnavailableWithNonCompliantLicense() throws new ActionListener() { @Override public void onResponse(final AcknowledgedResponse response) { + latch.countDown(); fail(); } @@ -69,6 +70,7 @@ public void testThatCreateAndFollowingIndexIsUnavailableWithNonCompliantLicense( new ActionListener() { @Override public void onResponse(final CreateAndFollowIndexAction.Response response) { + latch.countDown(); fail(); } @@ -86,6 +88,7 @@ public void testThatCcrStatsAreUnavailableWithNonCompliantLicense() throws Inter client().execute(CcrStatsAction.INSTANCE, new CcrStatsAction.TasksRequest(), new ActionListener() { @Override public void onResponse(final CcrStatsAction.TasksResponse tasksResponse) { + latch.countDown(); fail(); } From 6eca6274090f8a460d9381ef27d6f335bfdd738b Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Sun, 9 Sep 2018 10:22:22 -0400 Subject: [PATCH 11/12] Reverse logic for CCR license checks (#33549) This commit reverses the logic for CCR license checks in a few actions. This is done so that the successful case, which tends to be a larger block of code, does not require indentation. --- .../action/CreateAndFollowIndexAction.java | 28 +++++++++---------- .../xpack/ccr/action/FollowIndexAction.java | 28 +++++++++---------- .../ccr/action/TransportCcrStatsAction.java | 6 ++-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java index cf77bf8112f91..1e14eb8979fb7 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CreateAndFollowIndexAction.java @@ -221,21 +221,21 @@ protected Response newResponse() { @Override protected void masterOperation( final Request request, final ClusterState state, final ActionListener listener) throws Exception { - if (ccrLicenseChecker.isCcrAllowed()) { - final String[] indices = new String[]{request.getFollowRequest().getLeaderIndex()}; - final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); - if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { - createFollowerIndexAndFollowLocalIndex(request, state, listener); - } else { - assert remoteClusterIndices.size() == 1; - final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); - assert entry.getValue().size() == 1; - final String clusterAlias = entry.getKey(); - final String leaderIndex = entry.getValue().get(0); - createFollowerIndexAndFollowRemoteIndex(request, clusterAlias, leaderIndex, listener); - } - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } + final String[] indices = new String[]{request.getFollowRequest().getLeaderIndex()}; + final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); + if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + createFollowerIndexAndFollowLocalIndex(request, state, listener); + } else { + assert remoteClusterIndices.size() == 1; + final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); + assert entry.getValue().size() == 1; + final String clusterAlias = entry.getKey(); + final String leaderIndex = entry.getValue().get(0); + createFollowerIndexAndFollowRemoteIndex(request, clusterAlias, leaderIndex, listener); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java index 2a14c4e9a50bb..498224551106d 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/FollowIndexAction.java @@ -328,21 +328,21 @@ public TransportAction( protected void doExecute(final Task task, final Request request, final ActionListener listener) { - if (ccrLicenseChecker.isCcrAllowed()) { - final String[] indices = new String[]{request.leaderIndex}; - final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); - if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { - followLocalIndex(request, listener); - } else { - assert remoteClusterIndices.size() == 1; - final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); - assert entry.getValue().size() == 1; - final String clusterAlias = entry.getKey(); - final String leaderIndex = entry.getValue().get(0); - followRemoteIndex(request, clusterAlias, leaderIndex, listener); - } - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; + } + final String[] indices = new String[]{request.leaderIndex}; + final Map> remoteClusterIndices = remoteClusterService.groupClusterIndices(indices, s -> false); + if (remoteClusterIndices.containsKey(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + followLocalIndex(request, listener); + } else { + assert remoteClusterIndices.size() == 1; + final Map.Entry> entry = remoteClusterIndices.entrySet().iterator().next(); + assert entry.getValue().size() == 1; + final String clusterAlias = entry.getKey(); + final String leaderIndex = entry.getValue().get(0); + followRemoteIndex(request, clusterAlias, leaderIndex, listener); } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java index 33873201f5fb3..3b5d0ac53cf81 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportCcrStatsAction.java @@ -65,11 +65,11 @@ protected void doExecute( final Task task, final CcrStatsAction.TasksRequest request, final ActionListener listener) { - if (ccrLicenseChecker.isCcrAllowed()) { - super.doExecute(task, request, listener); - } else { + if (ccrLicenseChecker.isCcrAllowed() == false) { listener.onFailure(LicenseUtils.newComplianceException("ccr")); + return; } + super.doExecute(task, request, listener); } @Override From d4b212c4c9f8f153522ef4c1d8f0c588bb5fc3d6 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Sun, 9 Sep 2018 17:31:02 +0200 Subject: [PATCH 12/12] CORE: Make Pattern Exclusion Work with Aliases (#33518) * CORE: Make Pattern Exclusion Work with Aliases * Adds the pattern exclusion logic to finding aliases * Closes #33395 --- .../cluster/metadata/MetaData.java | 37 +++++++++++++------ .../cluster/metadata/MetaDataTests.java | 32 ++++++++++++++++ 2 files changed, 58 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java index c3da63886140a..75869b54850d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java @@ -296,14 +296,38 @@ private ImmutableOpenMap> findAliases(String[] origi return ImmutableOpenMap.of(); } - boolean matchAllAliases = matchAllAliases(aliases); + String[] patterns = new String[aliases.length]; + boolean[] include = new boolean[aliases.length]; + for (int i = 0; i < aliases.length; i++) { + String alias = aliases[i]; + if (alias.charAt(0) == '-') { + patterns[i] = alias.substring(1); + include[i] = false; + } else { + patterns[i] = alias; + include[i] = true; + } + } + boolean matchAllAliases = patterns.length == 0; ImmutableOpenMap.Builder> mapBuilder = ImmutableOpenMap.builder(); for (String index : concreteIndices) { IndexMetaData indexMetaData = indices.get(index); List filteredValues = new ArrayList<>(); for (ObjectCursor cursor : indexMetaData.getAliases().values()) { AliasMetaData value = cursor.value; - if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) { + boolean matched = matchAllAliases; + String alias = value.alias(); + for (int i = 0; i < patterns.length; i++) { + if (include[i]) { + if (matched == false) { + String pattern = patterns[i]; + matched = ALL.equals(pattern) || Regex.simpleMatch(pattern, alias); + } + } else if (matched) { + matched = Regex.simpleMatch(patterns[i], alias) == false; + } + } + if (matched) { filteredValues.add(value); } } @@ -317,15 +341,6 @@ private ImmutableOpenMap> findAliases(String[] origi return mapBuilder.build(); } - private static boolean matchAllAliases(final String[] aliases) { - for (String alias : aliases) { - if (alias.equals(ALL)) { - return true; - } - } - return aliases.length == 0; - } - /** * Checks if at least one of the specified aliases exists in the specified concrete indices. Wildcards are supported in the * alias names for partial matches. diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java index 9d82e9e1cdca5..da50e99705dfb 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java @@ -109,6 +109,38 @@ public void testFindAliases() { } } + public void testFindAliasWithExclusion() { + MetaData metaData = MetaData.builder().put( + IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("alias1").build()) + .putAlias(AliasMetaData.builder("alias2").build()) + ).build(); + List aliases = + metaData.findAliases(new GetAliasesRequest().aliases("*", "-alias1"), new String[] {"index"}).get("index"); + assertThat(aliases.size(), equalTo(1)); + assertThat(aliases.get(0).alias(), equalTo("alias2")); + } + + public void testFindAliasWithExclusionAndOverride() { + MetaData metaData = MetaData.builder().put( + IndexMetaData.builder("index") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .putAlias(AliasMetaData.builder("aa").build()) + .putAlias(AliasMetaData.builder("ab").build()) + .putAlias(AliasMetaData.builder("bb").build()) + ).build(); + List aliases = + metaData.findAliases(new GetAliasesRequest().aliases("a*", "-*b", "b*"), new String[] {"index"}).get("index"); + assertThat(aliases.size(), equalTo(2)); + assertThat(aliases.get(0).alias(), equalTo("aa")); + assertThat(aliases.get(1).alias(), equalTo("bb")); + } + public void testIndexAndAliasWithSameName() { IndexMetaData.Builder builder = IndexMetaData.builder("index") .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT))