diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 011da53384d5d..493ca9c2f04b5 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -38,7 +38,6 @@ import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.junit.AfterClass; import org.junit.Before; @@ -184,19 +183,44 @@ public static Iterable createParameters() throws Exception { */ public static Iterable createParameters(NamedXContentRegistry executeableSectionRegistry) throws Exception { String[] paths = resolvePathsProperty(REST_TESTS_SUITE, ""); // default to all tests under the test root - List tests = new ArrayList<>(); Map> yamlSuites = loadSuites(paths); + List suites = new ArrayList<>(); + IllegalArgumentException validationException = null; // yaml suites are grouped by directory (effectively by api) for (String api : yamlSuites.keySet()) { List yamlFiles = new ArrayList<>(yamlSuites.get(api)); for (Path yamlFile : yamlFiles) { - ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile); - for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) { - tests.add(new Object[]{ new ClientYamlTestCandidate(restTestSuite, testSection) }); + ClientYamlTestSuite suite = ClientYamlTestSuite.parse(executeableSectionRegistry, api, yamlFile); + suites.add(suite); + try { + suite.validate(); + } catch(Exception e) { + if (validationException == null) { + validationException = new IllegalArgumentException("Validation errors for the following test suites:\n- " + + e.getMessage()); + } else { + String previousMessage = validationException.getMessage(); + Throwable[] suppressed = validationException.getSuppressed(); + validationException = new IllegalArgumentException(previousMessage + "\n- " + e.getMessage()); + for (Throwable t : suppressed) { + validationException.addSuppressed(t); + } + } + validationException.addSuppressed(e); } } } + if (validationException != null) { + throw validationException; + } + + List tests = new ArrayList<>(); + for (ClientYamlTestSuite yamlTestSuite : suites) { + for (ClientYamlTestSection testSection : yamlTestSuite.getTestSections()) { + tests.add(new Object[]{ new ClientYamlTestCandidate(yamlTestSuite, testSection) }); + } + } //sort the candidates so they will always be in the same order before being shuffled, for repeatability tests.sort(Comparator.comparing(o -> ((ClientYamlTestCandidate) o[0]).getTestPath())); return tests; @@ -361,7 +385,7 @@ public void test() throws IOException { } } finally { logger.debug("start teardown test [{}]", testCandidate.getTestPath()); - for (DoSection doSection : testCandidate.getTeardownSection().getDoSections()) { + for (ExecutableSection doSection : testCandidate.getTeardownSection().getDoSections()) { executeSection(doSection); } logger.debug("end teardown test [{}]", testCandidate.getTestPath()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 1ec2382fac596..53988f8dd52a2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -18,13 +18,13 @@ */ package org.elasticsearch.test.rest.yaml.section; -import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -33,34 +33,37 @@ public class ClientYamlTestSection implements Comparable { public static ClientYamlTestSection parse(XContentParser parser) throws IOException { ParserUtils.advanceToFieldName(parser); - ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName()); + XContentLocation sectionLocation = parser.getTokenLocation(); + String sectionName = parser.currentName(); + List executableSections = new ArrayList<>(); try { parser.nextToken(); - testSection.setSkipSection(SkipSection.parseIfNext(parser)); + SkipSection skipSection = SkipSection.parseIfNext(parser); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); - testSection.addExecutableSection(ExecutableSection.parse(parser)); + executableSections.add(ExecutableSection.parse(parser)); } if (parser.nextToken() != XContentParser.Token.END_OBJECT) { - throw new IllegalArgumentException("malformed section [" + testSection.getName() + "] expected [" + throw new IllegalArgumentException("malformed section [" + sectionName + "] expected [" + XContentParser.Token.END_OBJECT + "] but was [" + parser.currentToken() + "]"); } parser.nextToken(); - return testSection; + return new ClientYamlTestSection(sectionLocation, sectionName, skipSection, Collections.unmodifiableList(executableSections)); } catch (Exception e) { - throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e); + throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + sectionName + "]", e); } } private final XContentLocation location; private final String name; - private SkipSection skipSection; + private final SkipSection skipSection; private final List executableSections; - public ClientYamlTestSection(XContentLocation location, String name) { + ClientYamlTestSection(XContentLocation location, String name, SkipSection skipSection, List executableSections) { this.location = location; this.name = name; - this.executableSections = new ArrayList<>(); + this.skipSection = skipSection; + this.executableSections = executableSections; } public XContentLocation getLocation() { @@ -75,33 +78,10 @@ public SkipSection getSkipSection() { return skipSection; } - public void setSkipSection(SkipSection skipSection) { - this.skipSection = skipSection; - } - public List getExecutableSections() { return executableSections; } - public void addExecutableSection(ExecutableSection executableSection) { - if (executableSection instanceof DoSection) { - DoSection doSection = (DoSection) executableSection; - if (false == doSection.getExpectedWarningHeaders().isEmpty() - && false == skipSection.getFeatures().contains("warnings")) { - throw new IllegalArgumentException("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so " - + "runners that do not support the [warnings] section can skip the test at line [" - + doSection.getLocation().lineNumber + "]"); - } - if (NodeSelector.ANY != doSection.getApiCallSection().getNodeSelector() - && false == skipSection.getFeatures().contains("node_selector")) { - throw new IllegalArgumentException("Attempted to add a [do] with a [node_selector] section without a corresponding " - + "[skip] so runners that do not support the [node_selector] section can skip the test at line [" - + doSection.getLocation().lineNumber + "]"); - } - } - this.executableSections.add(executableSection); - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index 85796494ba964..84d331c4a171c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -32,6 +33,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.TreeSet; @@ -79,11 +81,10 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa "expected token to be START_OBJECT but was " + parser.currentToken()); } - ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName); - - restTestSuite.setSetupSection(SetupSection.parseIfNext(parser)); - restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser)); + SetupSection setupSection = SetupSection.parseIfNext(parser); + TeardownSection teardownSection = TeardownSection.parseIfNext(parser); + Set testSections = new TreeSet<>(); while(true) { //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed //we need to somehow distinguish between a null in the middle of a test ("---") @@ -93,27 +94,29 @@ public static ClientYamlTestSuite parse(String api, String suiteName, XContentPa break; } } - ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); - if (!restTestSuite.addTestSection(testSection)) { + if (testSections.add(testSection) == false) { throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]"); } } - return restTestSuite; + return new ClientYamlTestSuite(api, suiteName, setupSection, teardownSection, + Collections.unmodifiableList(new ArrayList<>(testSections))); } private final String api; private final String name; + private final SetupSection setupSection; + private final TeardownSection teardownSection; + private final List testSections; - private SetupSection setupSection; - private TeardownSection teardownSection; - - private Set testSections = new TreeSet<>(); - - public ClientYamlTestSuite(String api, String name) { + ClientYamlTestSuite(String api, String name, SetupSection setupSection, TeardownSection teardownSection, + List testSections) { this.api = api; this.name = name; + this.setupSection = setupSection; + this.teardownSection = teardownSection; + this.testSections = testSections; } public String getApi() { @@ -132,27 +135,52 @@ public SetupSection getSetupSection() { return setupSection; } - public void setSetupSection(SetupSection setupSection) { - this.setupSection = setupSection; - } - public TeardownSection getTeardownSection() { return teardownSection; } - public void setTeardownSection(TeardownSection teardownSection) { - this.teardownSection = teardownSection; + public void validate() { + validateExecutableSections(this, setupSection.getExecutableSections(), null, setupSection, null); + validateExecutableSections(this, teardownSection.getDoSections(), null, null, teardownSection); + for (ClientYamlTestSection testSection : testSections) { + validateExecutableSections(this, testSection.getExecutableSections(), testSection, setupSection, teardownSection); + } + } + + private static void validateExecutableSections(ClientYamlTestSuite yamlTestSuite, List sections, + ClientYamlTestSection testSection, + SetupSection setupSection, TeardownSection teardownSection) { + for (ExecutableSection section : sections) { + if (section instanceof DoSection) { + DoSection doSection = (DoSection) section; + if (false == doSection.getExpectedWarningHeaders().isEmpty() + && false == hasSkipFeature("warnings", testSection, setupSection, teardownSection)) { + throw new IllegalArgumentException(yamlTestSuite.getPath() + ": attempted to add a [do] with a [warnings] section " + + "without a corresponding [\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] " + + "section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); + } + if (NodeSelector.ANY != doSection.getApiCallSection().getNodeSelector() + && false == hasSkipFeature("node_selector", testSection, setupSection, teardownSection)) { + throw new IllegalArgumentException(yamlTestSuite.getPath() + ": attempted to add a [do] with a [node_selector] " + + "section without a corresponding [\"skip\": \"features\": \"node_selector\"] so runners that do not support the " + + "[node_selector] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); + } + } + } + } + + private static boolean hasSkipFeature(String feature, ClientYamlTestSection testSection, + SetupSection setupSection, TeardownSection teardownSection) { + return (testSection != null && hasSkipFeature(feature, testSection.getSkipSection())) || + (setupSection != null && hasSkipFeature(feature, setupSection.getSkipSection())) || + (teardownSection != null && hasSkipFeature(feature, teardownSection.getSkipSection())); } - /** - * Adds a {@link org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection} to the REST suite - * @return true if the test section was not already present, false otherwise - */ - public boolean addTestSection(ClientYamlTestSection testSection) { - return this.testSections.add(testSection); + private static boolean hasSkipFeature(String feature, SkipSection skipSection) { + return skipSection != null && skipSection.getFeatures().contains(feature); } public List getTestSections() { - return new ArrayList<>(testSections); + return testSections; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 5fb5c1d003dd4..dcda4a2a025e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -182,7 +182,6 @@ public static DoSection parse(XContentParser parser) throws IOException { return doSection; } - private static final Logger logger = LogManager.getLogger(DoSection.class); private final XContentLocation location; @@ -206,7 +205,7 @@ public ApiCallSection getApiCallSection() { return apiCallSection; } - public void setApiCallSection(ApiCallSection apiCallSection) { + void setApiCallSection(ApiCallSection apiCallSection) { this.apiCallSection = apiCallSection; } @@ -214,7 +213,7 @@ public void setApiCallSection(ApiCallSection apiCallSection) { * Warning headers that we expect from this response. If the headers don't match exactly this request is considered to have failed. * Defaults to emptyList. */ - public List getExpectedWarningHeaders() { + List getExpectedWarningHeaders() { return expectedWarningHeaders; } @@ -222,7 +221,7 @@ public List getExpectedWarningHeaders() { * Set the warning headers that we expect from this response. If the headers don't match exactly this request is considered to have * failed. Defaults to emptyList. */ - public void setExpectedWarningHeaders(List expectedWarningHeaders) { + void setExpectedWarningHeaders(List expectedWarningHeaders) { this.expectedWarningHeaders = expectedWarningHeaders; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java index 692888a003857..0729bd172c42f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -31,7 +32,7 @@ public class SetupSection { /** * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}. */ - public static SetupSection parseIfNext(XContentParser parser) throws IOException { + static SetupSection parseIfNext(XContentParser parser) throws IOException { ParserUtils.advanceToFieldName(parser); if ("setup".equals(parser.currentName())) { @@ -45,58 +46,42 @@ public static SetupSection parseIfNext(XContentParser parser) throws IOException } public static SetupSection parse(XContentParser parser) throws IOException { - SetupSection setupSection = new SetupSection(); - setupSection.setSkipSection(SkipSection.parseIfNext(parser)); - + SkipSection skipSection = SkipSection.parseIfNext(parser); + List executableSections = new ArrayList<>(); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); if ("do".equals(parser.currentName())) { - setupSection.addDoSection(DoSection.parse(parser)); + executableSections.add(DoSection.parse(parser)); } else if ("set".equals(parser.currentName())) { - setupSection.addSetSection(SetSection.parse(parser)); + executableSections.add(SetSection.parse(parser)); } else { throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section"); } parser.nextToken(); } - parser.nextToken(); - - return setupSection; + return new SetupSection(skipSection, Collections.unmodifiableList(executableSections)); } - public static final SetupSection EMPTY; + public static final SetupSection EMPTY = new SetupSection(SkipSection.EMPTY, Collections.emptyList()); - static { - EMPTY = new SetupSection(); - EMPTY.setSkipSection(SkipSection.EMPTY); - } + private final SkipSection skipSection; + private final List executableSections; - private SkipSection skipSection; - - private List executableSections = new ArrayList<>(); + SetupSection(SkipSection skipSection, List executableSections) { + this.skipSection = skipSection; + this.executableSections = executableSections; + } public SkipSection getSkipSection() { return skipSection; } - public void setSkipSection(SkipSection skipSection) { - this.skipSection = skipSection; - } - public List getExecutableSections() { return executableSections; } - public void addDoSection(DoSection doSection) { - this.executableSections.add(doSection); - } - - public void addSetSection(SetSection setSection) { - this.executableSections.add(setSection); - } - public boolean isEmpty() { return EMPTY.equals(this); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java index 3b272fe673c76..bc7faab590283 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java @@ -24,13 +24,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class TeardownSection { /** * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}. */ - public static TeardownSection parseIfNext(XContentParser parser) throws IOException { + static TeardownSection parseIfNext(XContentParser parser) throws IOException { ParserUtils.advanceToFieldName(parser); if ("teardown".equals(parser.currentName())) { @@ -44,50 +45,40 @@ public static TeardownSection parseIfNext(XContentParser parser) throws IOExcept } public static TeardownSection parse(XContentParser parser) throws IOException { - TeardownSection teardownSection = new TeardownSection(); - teardownSection.setSkipSection(SkipSection.parseIfNext(parser)); - + SkipSection skipSection = SkipSection.parseIfNext(parser); + List executableSections = new ArrayList<>(); while (parser.currentToken() != XContentParser.Token.END_ARRAY) { ParserUtils.advanceToFieldName(parser); if (!"do".equals(parser.currentName())) { throw new ParsingException(parser.getTokenLocation(), "section [" + parser.currentName() + "] not supported within teardown section"); } - - teardownSection.addDoSection(DoSection.parse(parser)); + executableSections.add(DoSection.parse(parser)); parser.nextToken(); } parser.nextToken(); - return teardownSection; + return new TeardownSection(skipSection, Collections.unmodifiableList(executableSections)); } - public static final TeardownSection EMPTY; + public static final TeardownSection EMPTY = new TeardownSection(SkipSection.EMPTY, Collections.emptyList()); - static { - EMPTY = new TeardownSection(); - EMPTY.setSkipSection(SkipSection.EMPTY); - } + private final SkipSection skipSection; + private final List doSections; - private SkipSection skipSection; - private List doSections = new ArrayList<>(); + TeardownSection(SkipSection skipSection, List doSections) { + this.skipSection = skipSection; + this.doSections = doSections; + } public SkipSection getSkipSection() { return skipSection; } - public void setSkipSection(SkipSection skipSection) { - this.skipSection = skipSection; - } - - public List getDoSections() { + public List getDoSections() { return doSections; } - public void addDoSection(DoSection doSection) { - this.doSections.add(doSection); - } - public boolean isEmpty() { return EMPTY.equals(this); } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index 500cff893cb1f..66a94beddd8ce 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -20,78 +20,19 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; -import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import java.io.IOException; import java.util.Map; -import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase { - public void testAddingDoWithoutSkips() { - int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(SkipSection.EMPTY); - DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); - doSection.setApiCallSection(new ApiCallSection("test")); - section.addExecutableSection(doSection); - } - - public void testAddingDoWithWarningWithSkip() { - int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(new SkipSection(null, singletonList("warnings"), null)); - DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); - doSection.setExpectedWarningHeaders(singletonList("foo")); - doSection.setApiCallSection(new ApiCallSection("test")); - section.addExecutableSection(doSection); - } - - public void testAddingDoWithWarningWithSkipButNotWarnings() { - int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); - DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); - doSection.setExpectedWarningHeaders(singletonList("foo")); - doSection.setApiCallSection(new ApiCallSection("test")); - Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection)); - assertEquals("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so runners that do not support the" - + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage()); - } - - public void testAddingDoWithNodeSelectorWithSkip() { - int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(new SkipSection(null, singletonList("node_selector"), null)); - DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); - ApiCallSection apiCall = new ApiCallSection("test"); - apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); - doSection.setApiCallSection(apiCall); - section.addExecutableSection(doSection); - } - - public void testAddingDoWithNodeSelectorWithSkipButNotWarnings() { - int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); - DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); - ApiCallSection apiCall = new ApiCallSection("test"); - apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); - doSection.setApiCallSection(apiCall); - Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection)); - assertEquals("Attempted to add a [do] with a [node_selector] section without a corresponding" - + " [skip] so runners that do not support the [node_selector] section can skip the test at" - + " line [" + lineNumber + "]", e.getMessage()); - } - public void testWrongIndentation() throws Exception { { XContentParser parser = createParser(YamlXContent.yamlXContent, @@ -297,7 +238,7 @@ public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception { LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6); assertThat(lengthAssertion.getField(), equalTo("_index")); assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6)); + assertThat(lengthAssertion.getExpectedValue(), equalTo(6)); IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7); assertThat(falseAssertion.getField(), equalTo("whatever")); @@ -305,12 +246,12 @@ public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception { GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8); assertThat(greaterThanAssertion.getField(), equalTo("size")); assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5)); + assertThat(greaterThanAssertion.getExpectedValue(), equalTo(5)); LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9); assertThat(lessThanAssertion.getField(), equalTo("size")); assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10)); + assertThat(lessThanAssertion.getExpectedValue(), equalTo(10)); } public void testSmallSection() throws Exception { diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java index 873702f7c68ce..460e7ca5d6a6d 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java @@ -20,11 +20,16 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import java.util.Collections; import java.util.Map; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -106,9 +111,11 @@ public void testParseTestSetupTeardownAndSections() throws Exception { assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(false)); assertThat(restTestSuite.getTeardownSection().getSkipSection().isEmpty(), equalTo(true)); assertThat(restTestSuite.getTeardownSection().getDoSections().size(), equalTo(1)); - assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getApi(), equalTo("indices.delete")); - assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().size(), equalTo(1)); - assertThat(restTestSuite.getTeardownSection().getDoSections().get(0).getApiCallSection().getParams().get("index"), + assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getApi(), + equalTo("indices.delete")); + assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().size(), + equalTo(1)); + assertThat(((DoSection)restTestSuite.getTeardownSection().getDoSections().get(0)).getApiCallSection().getParams().get("index"), equalTo("test_index")); } else { assertThat(restTestSuite.getTeardownSection().isEmpty(), equalTo(true)); @@ -378,4 +385,144 @@ public void testParseTestDuplicateTestSections() throws Exception { ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser)); assertThat(e.getMessage(), containsString("duplicate test section")); } + + public void testAddingDoWithoutSkips() { + int lineNumber = between(1, 10000); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + doSection.setApiCallSection(new ApiCallSection("test")); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.singletonList(doSection)); + ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", SetupSection.EMPTY, TeardownSection.EMPTY, + Collections.singletonList(section)); + clientYamlTestSuite.validate(); + } + + public void testAddingDoWithWarningWithoutSkipWarnings() { + int lineNumber = between(1, 10000); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + doSection.setExpectedWarningHeaders(singletonList("foo")); + doSection.setApiCallSection(new ApiCallSection("test")); + ClientYamlTestSuite testSuite = createTestSuite(doSection); + Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); + assertEquals("api/name: attempted to add a [do] with a [warnings] section without a corresponding " + + "[\"skip\": \"features\": \"warnings\"] so runners that do not support the [warnings] section can skip the test " + + "at line [" + lineNumber + "]", e.getMessage()); + } + + public void testAddingDoWithNodeSelectorWithoutSkipNodeSelector() { + int lineNumber = between(1, 10000); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + ApiCallSection apiCall = new ApiCallSection("test"); + apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); + doSection.setApiCallSection(apiCall); + ClientYamlTestSuite testSuite = createTestSuite(doSection); + Exception e = expectThrows(IllegalArgumentException.class, testSuite::validate); + assertEquals("api/name: attempted to add a [do] with a [node_selector] section without a corresponding" + + " [\"skip\": \"features\": \"node_selector\"] so runners that do not support the [node_selector] section can skip the test at" + + " line [" + lineNumber + "]", e.getMessage()); + } + + private static ClientYamlTestSuite createTestSuite(DoSection doSection) { + final SetupSection setupSection; + final TeardownSection teardownSection; + final ClientYamlTestSection clientYamlTestSection; + switch(randomIntBetween(0, 2)) { + case 0: + setupSection = new SetupSection(SkipSection.EMPTY, Collections.singletonList(doSection)); + teardownSection = TeardownSection.EMPTY; + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.emptyList()); + break; + case 1: + setupSection = SetupSection.EMPTY; + teardownSection = new TeardownSection(SkipSection.EMPTY, Collections.singletonList(doSection)); + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.emptyList()); + break; + case 2: + setupSection = SetupSection.EMPTY; + teardownSection = TeardownSection.EMPTY; + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.singletonList(doSection)); + break; + default: + throw new UnsupportedOperationException(); + } + + return new ClientYamlTestSuite("api", "name", setupSection, teardownSection, + Collections.singletonList(clientYamlTestSection)); + } + + public void testAddingDoWithWarningWithSkip() { + int lineNumber = between(1, 10000); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + doSection.setExpectedWarningHeaders(singletonList("foo")); + doSection.setApiCallSection(new ApiCallSection("test")); + SkipSection skipSection = new SkipSection(null, singletonList("warnings"), null); + createTestSuiteAndValidate(skipSection, doSection); + } + + public void testAddingDoWithNodeSelectorWithSkip() { + int lineNumber = between(1, 10000); + SkipSection skipSection = new SkipSection(null, singletonList("node_selector"), null); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + ApiCallSection apiCall = new ApiCallSection("test"); + apiCall.setNodeSelector(NodeSelector.SKIP_DEDICATED_MASTERS); + doSection.setApiCallSection(apiCall); + createTestSuiteAndValidate(skipSection, doSection); + } + + public void testAddingDoWithHeadersWithSkip() { + int lineNumber = between(1, 10000); + SkipSection skipSection = new SkipSection(null, singletonList("headers"), null); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + ApiCallSection apiCallSection = new ApiCallSection("test"); + apiCallSection.addHeaders(singletonMap("foo", "bar")); + doSection.setApiCallSection(apiCallSection); + createTestSuiteAndValidate(skipSection, doSection); + } + + private static void createTestSuiteAndValidate(SkipSection skipSection, DoSection doSection) { + final SetupSection setupSection; + final TeardownSection teardownSection; + final ClientYamlTestSection clientYamlTestSection; + switch(randomIntBetween(0, 4)) { + case 0: + setupSection = new SetupSection(skipSection, Collections.emptyList()); + teardownSection = TeardownSection.EMPTY; + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.singletonList(doSection)); + break; + case 1: + setupSection = SetupSection.EMPTY; + teardownSection = new TeardownSection(skipSection, Collections.emptyList()); + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, Collections.singletonList(doSection)); + break; + case 2: + setupSection = SetupSection.EMPTY; + teardownSection = TeardownSection.EMPTY; + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + skipSection, Collections.singletonList(doSection)); + break; + case 3: + setupSection = new SetupSection(skipSection, Collections.singletonList(doSection)); + teardownSection = TeardownSection.EMPTY; + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(doSection)); + break; + case 4: + setupSection = SetupSection.EMPTY; + teardownSection = new TeardownSection(skipSection, Collections.singletonList(doSection)); + clientYamlTestSection = new ClientYamlTestSection(new XContentLocation(0, 0), "test", + SkipSection.EMPTY, randomBoolean() ? Collections.emptyList() : Collections.singletonList(doSection)); + + break; + default: + throw new UnsupportedOperationException(); + } + ClientYamlTestSuite clientYamlTestSuite = new ClientYamlTestSuite("api", "name", setupSection, teardownSection, + Collections.singletonList(clientYamlTestSection)); + clientYamlTestSuite.validate(); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java index 07afa9f33b5b1..96bff85389c8a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java @@ -49,8 +49,8 @@ public void testParseTeardownSection() throws Exception { assertThat(section, notNullValue()); assertThat(section.getSkipSection().isEmpty(), equalTo(true)); assertThat(section.getDoSections().size(), equalTo(2)); - assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete")); - assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2")); + assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete")); + assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2")); } public void testParseWithSkip() throws Exception { @@ -79,7 +79,7 @@ public void testParseWithSkip() throws Exception { assertThat(section.getSkipSection().getUpperVersion(), equalTo(Version.V_6_3_0)); assertThat(section.getSkipSection().getReason(), equalTo("there is a reason")); assertThat(section.getDoSections().size(), equalTo(2)); - assertThat(section.getDoSections().get(0).getApiCallSection().getApi(), equalTo("delete")); - assertThat(section.getDoSections().get(1).getApiCallSection().getApi(), equalTo("delete2")); + assertThat(((DoSection)section.getDoSections().get(0)).getApiCallSection().getApi(), equalTo("delete")); + assertThat(((DoSection)section.getDoSections().get(1)).getApiCallSection().getApi(), equalTo("delete2")); } }