From 8847c3409303bc8d4c2ff0107c2a34a5068fc4ef Mon Sep 17 00:00:00 2001 From: Nic Palmer Date: Wed, 21 Dec 2016 13:10:33 +0000 Subject: [PATCH 01/22] Push for eu-west-2 issue --- .../java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index 7eac1184fbfa3..f8dec51f41a1d 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -178,6 +178,9 @@ protected static String findEndpoint(Logger logger, Settings settings) { case "eu-west-1": endpoint = "ec2.eu-west-1.amazonaws.com"; break; + case "eu-west-2": + endpoint = "ec2.eu-west-2.amazonaws.com"; + break; case "eu-central": case "eu-central-1": endpoint = "ec2.eu-central-1.amazonaws.com"; From 3894ec9bae53d37744c2933c9b841982361a675b Mon Sep 17 00:00:00 2001 From: Nic Palmer Date: Wed, 21 Dec 2016 15:48:07 +0000 Subject: [PATCH 02/22] Fixed eu-west-2 entries for discovery-ec2 and repository-s3 also updated the asciidocs --- docs/plugins/discovery-ec2.asciidoc | 1 + docs/plugins/repository-s3.asciidoc | 1 + .../java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java | 2 +- .../java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 26d4f32189905..861a6da575b95 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -120,6 +120,7 @@ The available values are: * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo) * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul) * `eu-west` (`eu-west-1`) for EU (Ireland) +* `eu-west-2` (`eu-west-2`) for EU (London) * `eu-central` (`eu-central-1`) for EU (Frankfurt) * `sa-east` (`sa-east-1`) for South America (São Paulo) * `cn-north` (`cn-north-1`) for China (Beijing) diff --git a/docs/plugins/repository-s3.asciidoc b/docs/plugins/repository-s3.asciidoc index fcac31059306a..351d1daeca805 100644 --- a/docs/plugins/repository-s3.asciidoc +++ b/docs/plugins/repository-s3.asciidoc @@ -125,6 +125,7 @@ The available values are: * `ap-northeast` (`ap-northeast-1`) for Asia Pacific (Tokyo) * `ap-northeast-2` (`ap-northeast-2`) for Asia Pacific (Seoul) * `eu-west` (`eu-west-1`) for EU (Ireland) +* `eu-west-2` (`eu-west-2`) for EU (London) * `eu-central` (`eu-central-1`) for EU (Frankfurt) * `sa-east` (`sa-east-1`) for South America (São Paulo) * `cn-north` (`cn-north-1`) for China (Beijing) diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java index f8dec51f41a1d..fb76967914dca 100644 --- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java +++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/cloud/aws/AwsEc2ServiceImpl.java @@ -210,4 +210,4 @@ public void close() throws IOException { // Ensure that IdleConnectionReaper is shutdown IdleConnectionReaper.shutdown(); } -} +} \ No newline at end of file diff --git a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java index 25a8ab403d767..065d571dc359c 100644 --- a/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java +++ b/plugins/repository-s3/src/main/java/org/elasticsearch/cloud/aws/InternalAwsS3Service.java @@ -202,6 +202,9 @@ private static String getEndpoint(String region) { case "eu-west-1": endpoint = "s3-eu-west-1.amazonaws.com"; break; + case "eu-west-2": + endpoint = "s3-eu-west-2.amazonaws.com"; + break; case "eu-central": case "eu-central-1": endpoint = "s3.eu-central-1.amazonaws.com"; From 84edf36f11da2d9c416a14ae2420c56a32823a38 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 21 Dec 2016 16:51:45 +0100 Subject: [PATCH 03/22] Make `-0` compare less than `+0` consistently. (#22173) Our `float`/`double` fields generally assume that `-0` compares less than `+0`, except when bounds are exclusive: an exclusive lower bound on `-0` excludes `+0` and an exclusive upper bound on `+0` excludes `-0`. Closes #22167 --- .../index/mapper/NumberFieldMapper.java | 86 +++++++++++++++++-- .../index/mapper/NumberFieldTypeTests.java | 16 ++++ docs/reference/mapping/types/numeric.asciidoc | 6 ++ 3 files changed, 101 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 9e615e1016929..33a6d481ae428 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -186,6 +186,30 @@ Query termsQuery(String field, List values) { return HalfFloatPoint.newSetQuery(field, v); } + private float nextDown(float f) { + // HalfFloatPoint.nextDown considers that -0 is the same as +0 + // while point ranges are consistent with Float.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextDown(+0) returns -0 + if (Float.floatToIntBits(f) == Float.floatToIntBits(0f)) { + return -0f; + } else { + return HalfFloatPoint.nextDown(f); + } + } + + private float nextUp(float f) { + // HalfFloatPoint.nextUp considers that -0 is the same as +0 + // while point ranges are consistent with Float.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextUp(-0) returns +0 + if (Float.floatToIntBits(f) == Float.floatToIntBits(-0f)) { + return +0f; + } else { + return HalfFloatPoint.nextUp(f); + } + } + @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) { @@ -194,16 +218,16 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, if (lowerTerm != null) { l = parse(lowerTerm); if (includeLower) { - l = Math.nextDown(l); + l = nextDown(l); } l = HalfFloatPoint.nextUp(l); } if (upperTerm != null) { u = parse(upperTerm); if (includeUpper) { - u = Math.nextUp(u); + u = nextUp(u); } - u = HalfFloatPoint.nextDown(u); + u = nextDown(u); } return HalfFloatPoint.newRangeQuery(field, l, u); } @@ -276,6 +300,30 @@ Query termsQuery(String field, List values) { return FloatPoint.newSetQuery(field, v); } + private float nextDown(float f) { + // Math.nextDown considers that -0 is the same as +0 + // while point ranges are consistent with Float.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextDown(+0) returns -0 + if (Float.floatToIntBits(f) == Float.floatToIntBits(0f)) { + return -0f; + } else { + return Math.nextDown(f); + } + } + + private float nextUp(float f) { + // Math.nextUp considers that -0 is the same as +0 + // while point ranges are consistent with Float.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextUp(-0) returns +0 + if (Float.floatToIntBits(f) == Float.floatToIntBits(-0f)) { + return +0f; + } else { + return Math.nextUp(f); + } + } + @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) { @@ -284,13 +332,13 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, if (lowerTerm != null) { l = parse(lowerTerm); if (includeLower == false) { - l = Math.nextUp(l); + l = nextUp(l); } } if (upperTerm != null) { u = parse(upperTerm); if (includeUpper == false) { - u = Math.nextDown(u); + u = nextDown(u); } } return FloatPoint.newRangeQuery(field, l, u); @@ -364,6 +412,30 @@ Query termsQuery(String field, List values) { return DoublePoint.newSetQuery(field, v); } + private double nextDown(double d) { + // Math.nextDown considers that -0 is the same as +0 + // while point ranges are consistent with Double.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextDown(+0) returns -0 + if (Double.doubleToLongBits(d) == Double.doubleToLongBits(0d)) { + return -0d; + } else { + return Math.nextDown(d); + } + } + + private double nextUp(double d) { + // Math.nextUp considers that -0 is the same as +0 + // while point ranges are consistent with Double.compare, so + // they consider that -0 < +0, so we explicitly make sure + // that nextUp(-0) returns +0 + if (Double.doubleToLongBits(d) == Double.doubleToLongBits(-0d)) { + return +0d; + } else { + return Math.nextUp(d); + } + } + @Override Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) { @@ -372,13 +444,13 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, if (lowerTerm != null) { l = parse(lowerTerm); if (includeLower == false) { - l = Math.nextUp(l); + l = nextUp(l); } } if (upperTerm != null) { u = parse(upperTerm); if (includeUpper == false) { - u = Math.nextDown(u); + u = nextDown(u); } } return DoublePoint.newRangeQuery(field, l, u); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index d7e178404f158..19f0936de540f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -149,4 +149,20 @@ public void testHalfFloatRange() throws IOException { } IOUtils.close(reader, dir); } + + public void testNegativeZero() { + assertEquals( + NumberType.DOUBLE.rangeQuery("field", null, -0d, true, true), + NumberType.DOUBLE.rangeQuery("field", null, +0d, true, false)); + assertEquals( + NumberType.FLOAT.rangeQuery("field", null, -0f, true, true), + NumberType.FLOAT.rangeQuery("field", null, +0f, true, false)); + assertEquals( + NumberType.HALF_FLOAT.rangeQuery("field", null, -0f, true, true), + NumberType.HALF_FLOAT.rangeQuery("field", null, +0f, true, false)); + + assertFalse(NumberType.DOUBLE.termQuery("field", -0d).equals(NumberType.DOUBLE.termQuery("field", +0d))); + assertFalse(NumberType.FLOAT.termQuery("field", -0f).equals(NumberType.FLOAT.termQuery("field", +0f))); + assertFalse(NumberType.HALF_FLOAT.termQuery("field", -0f).equals(NumberType.HALF_FLOAT.termQuery("field", +0f))); + } } diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 6fdc0c806a4d8..48a74cd176034 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -39,6 +39,12 @@ PUT my_index -------------------------------------------------- // CONSOLE +NOTE: The `double`, `float` and `half_float` types consider that `-0.0` and +`+0.0` are different values. As a consequence, doing a `term` query on +`-0.0` will not match `+0.0` and vice-versa. Same is true for range queries: +if the upper bound is `-0.0` then `+0.0` will not match, and if the lower +bound is `+0.0` then `-0.0` will not match. + ==== Which type should I use? As far as integer types (`byte`, `short`, `integer` and `long`) are concerned, From 5c8232c03b8c7c0edbea74d8139f441083a6d439 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 21 Dec 2016 16:56:55 +0100 Subject: [PATCH 04/22] Restore deprecation warning for invalid match_mapping_type values (#22304) The deprecation warning gives now the same message as 5.x. The deprecation warning was previously removed, but given that we are still lenient with old indices we should still output the warning. --- .../java/org/elasticsearch/index/mapper/DynamicTemplate.java | 4 +++- .../org/elasticsearch/index/mapper/DynamicTemplateTests.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java index dcb991095b2dd..c31306c7fa38b 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java @@ -152,7 +152,7 @@ public static XContentFieldType fromString(String value) { return v; } } - throw new IllegalArgumentException("No xcontent type matched on [" + value + "], possible values are " + throw new IllegalArgumentException("No field type matched on [" + value + "], possible values are " + Arrays.toString(values())); } @@ -208,6 +208,8 @@ public static DynamicTemplate parse(String name, Map conf, if (indexVersionCreated.onOrAfter(Version.V_6_0_0_alpha1_UNRELEASED)) { throw e; } else { + DEPRECATION_LOGGER.deprecated("match_mapping_type [" + matchMappingType + "] is invalid and will be ignored: " + + e.getMessage()); // this template is on an unknown type so it will never match anything // null indicates that the template should be ignored return null; diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java index 24023281f5e7e..42dd8c3bf3752 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateTests.java @@ -50,14 +50,15 @@ public void testParseUnknownMatchType() throws IOException { templateDef.put("mapping", Collections.singletonMap("store", true)); // if a wrong match type is specified, we ignore the template assertNull(DynamicTemplate.parse("my_template", templateDef, Version.V_5_0_0_alpha5)); - + assertWarnings("match_mapping_type [short] is invalid and will be ignored: No field type matched on [short], " + + "possible values are [object, string, long, double, boolean, date, binary]"); Map templateDef2 = new HashMap<>(); templateDef2.put("match_mapping_type", "text"); templateDef2.put("mapping", Collections.singletonMap("store", true)); // if a wrong match type is specified, we ignore the template IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> DynamicTemplate.parse("my_template", templateDef2, Version.V_6_0_0_alpha1_UNRELEASED)); - assertEquals("No xcontent type matched on [text], possible values are [object, string, long, double, boolean, date, binary]", + assertEquals("No field type matched on [text], possible values are [object, string, long, double, boolean, date, binary]", e.getMessage()); } From dec6fc2d405d3efc5729e1bb7885fce10a62a477 Mon Sep 17 00:00:00 2001 From: Francesc Gil Date: Wed, 21 Dec 2016 17:31:21 +0100 Subject: [PATCH 05/22] Repeated language analyzers (#22240) * Repeated language analyzers The `catalan` analyzer was repeated on the supported list :) * Reordered the languages to have alphabetic order * Added space for format * Reordered the languages and removed repeated --- docs/reference/analysis/analyzers/lang-analyzer.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc index 983a719e4f477..02843789c4b3b 100644 --- a/docs/reference/analysis/analyzers/lang-analyzer.asciidoc +++ b/docs/reference/analysis/analyzers/lang-analyzer.asciidoc @@ -55,9 +55,9 @@ functionality is implemented by adding the with the `keywords` set to the value of the `stem_exclusion` parameter. The following analyzers support setting custom `stem_exclusion` list: -`arabic`, `armenian`, `basque`, `catalan`, `bulgarian`, `catalan`, -`czech`, `finnish`, `dutch`, `english`, `finnish`, `french`, `galician`, -`german`, `irish`, `hindi`, `hungarian`, `indonesian`, `italian`, `latvian`, +`arabic`, `armenian`, `basque`, `bulgarian`, `catalan`, `czech`, +`dutch`, `english`, `finnish`, `french`, `galician`, +`german`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `latvian`, `lithuanian`, `norwegian`, `portuguese`, `romanian`, `russian`, `sorani`, `spanish`, `swedish`, `turkish`. From 8aca504c863d2351258e33e86b2c18e9365bbf99 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Dec 2016 13:39:28 -0500 Subject: [PATCH 06/22] Clear static variable after suite This was causing test failures: https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+java9-periodic/1101/console https://elasticsearch-ci.elastic.co/job/elastic+elasticsearch+master+dockeralpine-periodic/513/consoleFull --- .../elasticsearch/index/query/QueryParseContextTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java index 1e00176f558d9..c509468a1a7fe 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryParseContextTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -45,6 +46,11 @@ public static void init() { xContentRegistry = new NamedXContentRegistry(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents()); } + @AfterClass + public static void cleanup() { + xContentRegistry = null; + } + private ThreadContext threadContext; @Before From 91cb56324726f7541dabb4dfe1e7aee103cac8bb Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 21 Dec 2016 22:37:07 -0500 Subject: [PATCH 07/22] Provide helpful error message if a plugin exists Today if an older version of a plugin exists, we fail to notify the user with a helpful error message. This happens because during plugin verification, we attempt to read the plugin descriptors for all existing plugins. When an older version of a plugin is sitting on disk, we will attempt to read this old plugin descriptor and fail due to a version mismatch. This leads to an unhelpful error message. Instead, we should check for existence of the plugin as part of the verification phase, but before attempting to read plugin descriptors for existing plugins. This enables us to provide a helpful error message to the user. Relates #22305 --- .../plugins/InstallPluginCommand.java | 19 ++++++++++++------- .../plugins/InstallPluginCommandTests.java | 16 +++++++++++++--- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 403816aafa880..dea6fcba312d0 100644 --- a/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/core/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -418,6 +418,18 @@ private Path stagingDirectoryWithoutPosixPermissions(Path pluginsDir) throws IOE private PluginInfo verify(Terminal terminal, Path pluginRoot, boolean isBatch, Environment env) throws Exception { // read and validate the plugin descriptor PluginInfo info = PluginInfo.readFromProperties(pluginRoot); + + // checking for existing version of the plugin + final Path destination = env.pluginsFile().resolve(info.getName()); + if (Files.exists(destination)) { + final String message = String.format( + Locale.ROOT, + "plugin directory [%s] already exists; if you need to update the plugin, uninstall it first using command 'remove %s'", + destination.toAbsolutePath(), + info.getName()); + throw new UserException(ExitCodes.CONFIG, message); + } + terminal.println(VERBOSE, info.toString()); // don't let user install plugin as a module... @@ -471,14 +483,7 @@ private void install(Terminal terminal, boolean isBatch, Path tmpRoot, Environme try { PluginInfo info = verify(terminal, tmpRoot, isBatch, env); - final Path destination = env.pluginsFile().resolve(info.getName()); - if (Files.exists(destination)) { - throw new UserException( - ExitCodes.USAGE, - "plugin directory " + destination.toAbsolutePath() + - " already exists. To update the plugin, uninstall it first using 'remove " + info.getName() + "' command"); - } Path tmpBinDir = tmpRoot.resolve("bin"); if (Files.exists(tmpBinDir)) { diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index 96e64e5c8882f..5ea91eae394d1 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -60,19 +60,17 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.UserPrincipal; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @LuceneTestCase.SuppressFileSystems("*") @@ -673,6 +671,18 @@ public void testQuietFlagEnabled() throws Exception { assertThat(terminal.getOutput(), not(containsString("100%"))); } + public void testPluginAlreadyInstalled() throws Exception { + Tuple env = createEnv(fs, temp); + Path pluginDir = createPluginDir(temp); + String pluginZip = createPlugin("fake", pluginDir); + installPlugin(pluginZip, env.v1()); + final UserException e = expectThrows(UserException.class, () -> installPlugin(pluginZip, env.v1(), randomBoolean())); + assertThat( + e.getMessage(), + equalTo("plugin directory [" + env.v2().pluginsFile().resolve("fake") + "] already exists; " + + "if you need to update the plugin, uninstall it first using command 'remove fake'")); + } + private void installPlugin(MockTerminal terminal, boolean isBatch) throws Exception { Tuple env = createEnv(fs, temp); Path pluginDir = createPluginDir(temp); From 7946396fe682103929dd81c8d21dec1ad6fb3c95 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 21 Dec 2016 23:08:16 -0500 Subject: [PATCH 08/22] Introduce translog no-op As the translog evolves towards a full operations log as part of the sequence numbers push, there is a need for the translog to be able to represent operations for which a sequence number was assigned, but the operation did not mutate the index. Examples of how this can arise are operations that fail after the sequence number is assigned, and gaps in this history that arise when an operation is assigned a sequence number but the operation never completed (e.g., a node crash). It is important that these operations appear in the history so that they can be replicated and replayed during recovery as otherwise the history will be incomplete and local checkpoints will not be able to advance. This commit introduces a no-op to the translog to set the stage for these efforts. Relates #22291 --- .../common/io/stream/StreamOutput.java | 2 +- .../elasticsearch/index/engine/Engine.java | 65 ++++++- .../index/engine/InternalEngine.java | 44 ++++- .../index/engine/ShadowEngine.java | 5 + .../shard/TranslogRecoveryPerformer.java | 24 ++- .../index/translog/Translog.java | 100 +++++++++- .../cluster/node/stats/NodeStatsTests.java | 77 ++++---- .../action/update/UpdateRequestTests.java | 2 +- .../common/FieldMemoryStatsTests.java | 2 +- .../common/unit/ByteSizeValueTests.java | 10 +- .../common/unit/SizeValueTests.java | 8 +- .../common/unit/TimeValueTests.java | 8 +- .../zen/ElectMasterServiceTests.java | 3 +- .../discovery/zen/UnicastZenPingTests.java | 10 +- .../fieldstats/FieldStatsTests.java | 48 ++--- .../index/engine/InternalEngineTests.java | 45 +++++ .../index/fielddata/FieldDataStatsTests.java | 2 +- .../index/get/GetResultTests.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldTypeTests.java | 2 +- .../index/query/QueryShardContextTests.java | 2 +- .../GlobalCheckpointSyncActionTests.java | 6 +- .../suggest/stats/CompletionsStatsTests.java | 2 +- .../index/translog/TranslogTests.java | 182 ++++++++++++------ .../monitor/os/OsProbeTests.java | 2 +- .../monitor/os/OsStatsTests.java | 8 +- .../monitor/process/ProcessProbeTests.java | 2 +- .../nodesinfo/NodeInfoStreamingTests.java | 4 +- .../search/SearchRequestTests.java | 2 +- .../ShardSearchTransportRequestTests.java | 2 +- .../rescore/QueryRescoreBuilderTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../index/reindex/RoundTripTests.java | 4 +- .../geoip/GeoIpProcessorFactoryTests.java | 2 +- .../test/AbstractQueryTestCase.java | 2 +- .../org/elasticsearch/test/ESTestCase.java | 2 +- .../search/MockSearchServiceTests.java | 2 +- 37 files changed, 501 insertions(+), 190 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 4fc253cf45d21..3f72c80720247 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -328,7 +328,7 @@ public void writeString(String str) throws IOException { // make sure any possible char can fit into the buffer in any possible iteration // we need at most 3 bytes so we flush the buffer once we have less than 3 bytes // left before we start another iteration - if (offset > buffer.length-3) { + if (offset > buffer.length - 3) { writeBytes(buffer, offset); offset = 0; } diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index 0f1c05e59ee95..b081d76f3c74b 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -47,7 +47,6 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -296,6 +295,8 @@ public Condition newCondition() { */ public abstract DeleteResult delete(final Delete delete); + public abstract NoOpResult noOp(final NoOp noOp); + /** * Base class for index and delete operation results * Holds result meta data (e.g. translog location, updated version) @@ -382,6 +383,7 @@ void freeze() { } public static class IndexResult extends Result { + private final boolean created; public IndexResult(long version, long seqNo, boolean created) { @@ -397,9 +399,11 @@ public IndexResult(Exception failure, long version, long seqNo) { public boolean isCreated() { return created; } + } public static class DeleteResult extends Result { + private final boolean found; public DeleteResult(long version, long seqNo, boolean found) { @@ -415,6 +419,19 @@ public DeleteResult(Exception failure, long version, long seqNo) { public boolean isFound() { return found; } + + } + + static class NoOpResult extends Result { + + NoOpResult(long seqNo) { + super(Operation.TYPE.NO_OP, 0, seqNo); + } + + NoOpResult(long seqNo, Exception failure) { + super(Operation.TYPE.NO_OP, failure, 0, seqNo); + } + } /** @@ -910,7 +927,7 @@ public abstract static class Operation { /** type of operation (index, delete), subclasses use static types */ public enum TYPE { - INDEX, DELETE; + INDEX, DELETE, NO_OP; private final String lowercase; @@ -1114,6 +1131,50 @@ TYPE operationType() { public int estimatedSizeInBytes() { return (uid().field().length() + uid().text().length()) * 2 + 20; } + + } + + public static class NoOp extends Operation { + + private final String reason; + + public String reason() { + return reason; + } + + public NoOp( + final Term uid, + final long seqNo, + final long primaryTerm, + final long version, + final VersionType versionType, + final Origin origin, + final long startTime, + final String reason) { + super(uid, seqNo, primaryTerm, version, versionType, origin, startTime); + this.reason = reason; + } + + @Override + public String type() { + throw new UnsupportedOperationException(); + } + + @Override + String id() { + throw new UnsupportedOperationException(); + } + + @Override + TYPE operationType() { + return TYPE.NO_OP; + } + + @Override + public int estimatedSizeInBytes() { + return 2 * reason.length() + 2 * Long.BYTES; + } + } public static class Get { diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 2957b9ab06405..e142ac0f6f49e 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -277,7 +277,7 @@ private void recoverFromTranslog(TranslogRecoveryPerformer handler) throws IOExc } // flush if we recovered something or if we have references to older translogs // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0 length. - assert pendingTranslogRecovery.get(): "translogRecovery is not pending but should be"; + assert pendingTranslogRecovery.get() : "translogRecovery is not pending but should be"; pendingTranslogRecovery.set(false); // we are good - now we can commit if (opsRecovered > 0) { logger.trace("flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]", @@ -375,7 +375,7 @@ private static SeqNoStats loadSeqNoStatsFromLuceneAndTranslog( * specified global checkpoint. * * @param globalCheckpoint the global checkpoint to use - * @param indexWriter the index writer (for the Lucene commit point) + * @param indexWriter the index writer (for the Lucene commit point) * @return the sequence number stats */ private static SeqNoStats loadSeqNoStatsFromLucene(final long globalCheckpoint, final IndexWriter indexWriter) { @@ -434,7 +434,7 @@ public GetResult get(Get get, Function searcherFactory) throws if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) { Uid uid = Uid.createUid(get.uid().text()); throw new VersionConflictEngineException(shardId, uid.type(), uid.id(), - get.versionType().explainConflictForReads(versionValue.version(), get.version())); + get.versionType().explainConflictForReads(versionValue.version(), get.version())); } refresh("realtime_get"); } @@ -532,7 +532,7 @@ public IndexResult index(Index index) { * * @return failure if the failure is a document specific failure (e.g. analysis chain failure) * or throws Exception if the failure caused the engine to fail (e.g. out of disk, lucene tragic event) - * + *

* Note: pkg-private for testing */ final Exception checkIfDocumentFailureOrThrow(final Operation operation, final Exception failure) { @@ -577,7 +577,7 @@ private boolean canOptimizeAddDocument(Index index) { case PEER_RECOVERY: case REPLICA: assert index.version() == 1 && index.versionType() == VersionType.EXTERNAL - : "version: " + index.version() + " type: " + index.versionType(); + : "version: " + index.version() + " type: " + index.versionType(); return true; case LOCAL_TRANSLOG_RECOVERY: assert index.isRetry(); @@ -596,10 +596,10 @@ private boolean assertSequenceNumber(final Engine.Operation.Origin origin, final " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated() + ". seq no: " + seqNo; } else if (origin == Operation.Origin.PRIMARY) { // sequence number should not be set when operation origin is primary - assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "primary ops should never an assigned seq no. got: " + seqNo; + assert seqNo == SequenceNumbersService.UNASSIGNED_SEQ_NO : "primary ops should never have an assigned seq no. got: " + seqNo; } else { // sequence number should be set when operation origin is not primary - assert seqNo >= 0 : "replica ops should an assigned seq no. origin: " + origin + + assert seqNo >= 0 : "recovery or replica ops should have an assigned seq no. origin: " + origin + " index version: " + engineConfig.getIndexSettings().getIndexVersionCreated(); } return true; @@ -651,7 +651,7 @@ private IndexResult innerIndex(Index index) throws IOException { if (deOptimizeTimestamp >= index.getAutoGeneratedIdTimestamp()) { break; } - } while(maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp, + } while (maxUnsafeAutoIdTimestamp.compareAndSet(deOptimizeTimestamp, index.getAutoGeneratedIdTimestamp()) == false); assert maxUnsafeAutoIdTimestamp.get() >= index.getAutoGeneratedIdTimestamp(); } else { @@ -859,6 +859,34 @@ private boolean deleteIfFound(Term uid, long currentVersion, boolean deleted, Ve return found; } + @Override + public NoOpResult noOp(final NoOp noOp) { + NoOpResult noOpResult; + try (final ReleasableLock ignored = readLock.acquire()) { + noOpResult = innerNoOp(noOp); + } catch (final Exception e) { + noOpResult = new NoOpResult(noOp.seqNo(), e); + } + return noOpResult; + } + + private NoOpResult innerNoOp(final NoOp noOp) throws IOException { + assert noOp.seqNo() > SequenceNumbersService.NO_OPS_PERFORMED; + final long seqNo = noOp.seqNo(); + try { + final NoOpResult noOpResult = new NoOpResult(noOp.seqNo()); + final Translog.Location location = translog.add(new Translog.NoOp(noOp.seqNo(), noOp.primaryTerm(), noOp.reason())); + noOpResult.setTranslogLocation(location); + noOpResult.setTook(System.nanoTime() - noOp.startTime()); + noOpResult.freeze(); + return noOpResult; + } finally { + if (seqNo != SequenceNumbersService.UNASSIGNED_SEQ_NO) { + seqNoService().markSeqNoAsCompleted(seqNo); + } + } + } + @Override public void refresh(String source) throws EngineException { // we obtain a read lock here, since we don't want a flush to happen while we are refreshing diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java index 39bff89b7def0..c7226fc8b0dfc 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java @@ -116,6 +116,11 @@ public DeleteResult delete(Delete delete) { throw new UnsupportedOperationException(shardId + " delete operation not allowed on shadow engine"); } + @Override + public NoOpResult noOp(NoOp noOp) { + throw new UnsupportedOperationException(shardId + " no-op operation not allowed on shadow engine"); + } + @Override public SyncedFlushResult syncFlush(String syncId, CommitId expectedCommitId) { throw new UnsupportedOperationException(shardId + " sync commit operation not allowed on shadow engine"); diff --git a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java index f27958b71f5f9..b5c23d9dc467d 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java +++ b/core/src/main/java/org/elasticsearch/index/shard/TranslogRecoveryPerformer.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.engine.IgnoreOnRecoveryEngineException; import org.elasticsearch.index.mapper.DocumentMapperForType; @@ -29,6 +30,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.seqno.SequenceNumbersService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.rest.RestStatus; @@ -94,6 +96,7 @@ public int recoveryFromSnapshot(Engine engine, Translog.Snapshot snapshot) throw } } } + return opsRecovered; } @@ -158,22 +161,28 @@ private void performRecoveryOperation(Engine engine, Translog.Operation operatio .routing(index.routing()).parent(index.parent()), index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); - if (logger.isTraceEnabled()) { - logger.trace("[translog] recover [index] op of [{}][{}]", index.type(), index.id()); - } + logger.trace("[translog] recover [index] op [({}, {})] of [{}][{}]", index.seqNo(), index.primaryTerm(), index.type(), index.id()); index(engine, engineIndex); break; case DELETE: Translog.Delete delete = (Translog.Delete) operation; Uid uid = Uid.createUid(delete.uid().text()); - if (logger.isTraceEnabled()) { - logger.trace("[translog] recover [delete] op of [{}][{}]", uid.type(), uid.id()); - } + logger.trace("[translog] recover [delete] op [({}, {})] of [{}][{}]", delete.seqNo(), delete.primaryTerm(), uid.type(), uid.id()); final Engine.Delete engineDelete = new Engine.Delete(uid.type(), uid.id(), delete.uid(), delete.seqNo(), delete.primaryTerm(), delete.version(), delete.versionType().versionTypeForReplicationAndRecovery(), origin, System.nanoTime()); delete(engine, engineDelete); break; + case NO_OP: + final Translog.NoOp noOp = (Translog.NoOp) operation; + final long seqNo = noOp.seqNo(); + final long primaryTerm = noOp.primaryTerm(); + final String reason = noOp.reason(); + logger.trace("[translog] recover [no_op] op [({}, {})] of [{}]", seqNo, primaryTerm, reason); + final Engine.NoOp engineNoOp = + new Engine.NoOp(null, seqNo, primaryTerm, 0, VersionType.INTERNAL, origin, System.nanoTime(), reason); + noOp(engine, engineNoOp); + break; default: throw new IllegalStateException("No operation defined for [" + operation + "]"); } @@ -206,6 +215,9 @@ protected void delete(Engine engine, Engine.Delete engineDelete) { engine.delete(engineDelete); } + protected void noOp(Engine engine, Engine.NoOp engineNoOp) { + engine.noOp(engineNoOp); + } /** * Called once for every processed operation by this recovery performer. diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index f7560960660ca..4ca4f57e34114 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -55,6 +55,7 @@ import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -743,7 +744,8 @@ enum Type { @Deprecated CREATE((byte) 1), INDEX((byte) 2), - DELETE((byte) 3); + DELETE((byte) 3), + NO_OP((byte) 4); private final byte id; @@ -763,6 +765,8 @@ public static Type fromId(byte id) { return INDEX; case 3: return DELETE; + case 4: + return NO_OP; default: throw new IllegalArgumentException("No type mapped for [" + id + "]"); } @@ -786,9 +790,11 @@ static Operation readType(StreamInput input) throws IOException { // the deserialization logic in Index was identical to that of Create when create was deprecated return new Index(input); case DELETE: - return new Translog.Delete(input); + return new Delete(input); case INDEX: return new Index(input); + case NO_OP: + return new NoOp(input); default: throw new IOException("No type for [" + type + "]"); } @@ -805,6 +811,7 @@ static void writeType(Translog.Operation operation, StreamOutput output) throws } public static class Source { + public final BytesReference source; public final String routing; public final String parent; @@ -814,9 +821,11 @@ public Source(BytesReference source, String routing, String parent) { this.routing = routing; this.parent = parent; } + } public static class Index implements Operation { + public static final int FORMAT_2x = 6; // since 2.0-beta1 and 1.1 public static final int FORMAT_AUTO_GENERATED_IDS = 7; // since 5.0.0-beta1 public static final int FORMAT_SEQ_NO = FORMAT_AUTO_GENERATED_IDS + 1; // since 6.0.0 @@ -853,8 +862,8 @@ public Index(StreamInput in) throws IOException { this.autoGeneratedIdTimestamp = IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP; } if (format >= FORMAT_SEQ_NO) { - seqNo = in.readVLong(); - primaryTerm = in.readVLong(); + seqNo = in.readLong(); + primaryTerm = in.readLong(); } } @@ -943,11 +952,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(routing); out.writeOptionalString(parent); out.writeLong(version); - + out.writeByte(versionType.getValue()); out.writeLong(autoGeneratedIdTimestamp); - out.writeVLong(seqNo); - out.writeVLong(primaryTerm); + out.writeLong(seqNo); + out.writeLong(primaryTerm); } @Override @@ -1004,9 +1013,11 @@ public String toString() { public long getAutoGeneratedIdTimestamp() { return autoGeneratedIdTimestamp; } + } public static class Delete implements Operation { + private static final int FORMAT_5_X = 3; private static final int FORMAT_SEQ_NO = FORMAT_5_X + 1; public static final int SERIALIZATION_FORMAT = FORMAT_SEQ_NO; @@ -1127,6 +1138,81 @@ public String toString() { "uid=" + uid + '}'; } + + } + + public static class NoOp implements Operation { + + private final long seqNo; + private final long primaryTerm; + private final String reason; + + public long seqNo() { + return seqNo; + } + + public long primaryTerm() { + return primaryTerm; + } + + public String reason() { + return reason; + } + + NoOp(final StreamInput in) throws IOException { + seqNo = in.readLong(); + primaryTerm = in.readLong(); + reason = in.readString(); + } + + public NoOp(final long seqNo, final long primaryTerm, final String reason) { + assert seqNo > SequenceNumbersService.NO_OPS_PERFORMED; + assert primaryTerm >= 0; + assert reason != null; + this.seqNo = seqNo; + this.primaryTerm = primaryTerm; + this.reason = reason; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(seqNo); + out.writeLong(primaryTerm); + out.writeString(reason); + } + + @Override + public Type opType() { + return Type.NO_OP; + } + + @Override + public long estimateSize() { + return 2 * reason.length() + 2 * Long.BYTES; + } + + @Override + public Source getSource() { + throw new UnsupportedOperationException("source does not exist for a no-op"); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + final NoOp that = (NoOp) obj; + return seqNo == that.seqNo && primaryTerm == that.primaryTerm && reason.equals(that.reason); + } + + @Override + public int hashCode() { + return 31 * 31 * 31 + 31 * 31 * Long.hashCode(seqNo) + 31 * Long.hashCode(primaryTerm) + reason().hashCode(); + } + } diff --git a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index 6913abd81a4ae..d3d156ee2cdfd 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -291,41 +291,51 @@ private static NodeStats createNodeStats() { new OsStats.Swap(randomLong(), randomLong()), new OsStats.Cgroup( randomAsciiOfLength(8), - randomPositiveLong(), + randomNonNegativeLong(), randomAsciiOfLength(8), - randomPositiveLong(), - randomPositiveLong(), - new OsStats.Cgroup.CpuStat(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()))); + randomNonNegativeLong(), + randomNonNegativeLong(), + new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()))); } - ProcessStats processStats = frequently() ? new ProcessStats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - new ProcessStats.Cpu(randomShort(), randomPositiveLong()), - new ProcessStats.Mem(randomPositiveLong())) : null; + ProcessStats processStats = frequently() ? + new ProcessStats( + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), + new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()), + new ProcessStats.Mem(randomNonNegativeLong())) : + null; JvmStats jvmStats = null; if (frequently()) { int numMemoryPools = randomIntBetween(0, 10); List memoryPools = new ArrayList<>(numMemoryPools); for (int i = 0; i < numMemoryPools; i++) { - memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong())); + memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())); } JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000)); int numGarbageCollectors = randomIntBetween(0, 10); JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[numGarbageCollectors]; for (int i = 0; i < numGarbageCollectors; i++) { garbageCollectorsArray[i] = new JvmStats.GarbageCollector(randomAsciiOfLengthBetween(3, 10), - randomPositiveLong(), randomPositiveLong()); + randomNonNegativeLong(), randomNonNegativeLong()); } JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray); int numBufferPools = randomIntBetween(0, 10); List bufferPoolList = new ArrayList<>(); for (int i = 0; i < numBufferPools; i++) { - bufferPoolList.add(new JvmStats.BufferPool(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong())); + bufferPoolList.add( + new JvmStats.BufferPool( + randomAsciiOfLengthBetween(3, 10), + randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong())); } - JvmStats.Classes classes = new JvmStats.Classes(randomPositiveLong(), randomPositiveLong(), randomPositiveLong()); - jvmStats = frequently() ? new JvmStats(randomPositiveLong(), randomPositiveLong(), new JvmStats.Mem(randomPositiveLong(), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), memoryPools), threads, - garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null; + JvmStats.Classes classes = new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + jvmStats = + frequently() ? new JvmStats(randomNonNegativeLong(), randomNonNegativeLong(), new JvmStats.Mem(randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), memoryPools), + threads, garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null; } ThreadPoolStats threadPoolStats = null; if (frequently()) { @@ -333,7 +343,7 @@ private static NodeStats createNodeStats() { List threadPoolStatsList = new ArrayList<>(); for (int i = 0; i < numThreadPoolStats; i++) { threadPoolStatsList.add(new ThreadPoolStats.Stats(randomAsciiOfLengthBetween(3, 10), randomIntBetween(1, 1000), - randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomPositiveLong(), + randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomNonNegativeLong(), randomIntBetween(1, 1000), randomIntBetween(1, 1000))); } threadPoolStats = new ThreadPoolStats(threadPoolStatsList); @@ -345,50 +355,51 @@ private static NodeStats createNodeStats() { for (int i = 0; i < numDeviceStats; i++) { FsInfo.DeviceStats previousDeviceStats = randomBoolean() ? null : new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), null); - deviceStatsArray[i] = new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), previousDeviceStats); + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), null); + deviceStatsArray[i] = + new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), previousDeviceStats); } FsInfo.IoStats ioStats = new FsInfo.IoStats(deviceStatsArray); int numPaths = randomIntBetween(0, 10); FsInfo.Path[] paths = new FsInfo.Path[numPaths]; for (int i = 0; i < numPaths; i++) { paths[i] = new FsInfo.Path(randomAsciiOfLengthBetween(3, 10), randomBoolean() ? randomAsciiOfLengthBetween(3, 10) : null, - randomPositiveLong(), randomPositiveLong(), randomPositiveLong()); + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); } - fsInfo = new FsInfo(randomPositiveLong(), ioStats, paths); + fsInfo = new FsInfo(randomNonNegativeLong(), ioStats, paths); } - TransportStats transportStats = frequently() ? new TransportStats(randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong()) : null; - HttpStats httpStats = frequently() ? new HttpStats(randomPositiveLong(), randomPositiveLong()) : null; + TransportStats transportStats = frequently() ? new TransportStats(randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null; + HttpStats httpStats = frequently() ? new HttpStats(randomNonNegativeLong(), randomNonNegativeLong()) : null; AllCircuitBreakerStats allCircuitBreakerStats = null; if (frequently()) { int numCircuitBreakerStats = randomIntBetween(0, 10); CircuitBreakerStats[] circuitBreakerStatsArray = new CircuitBreakerStats[numCircuitBreakerStats]; for (int i = 0; i < numCircuitBreakerStats; i++) { - circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomPositiveLong(), - randomPositiveLong(), randomDouble(), randomPositiveLong()); + circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), + randomNonNegativeLong(), randomDouble(), randomNonNegativeLong()); } allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray); } - ScriptStats scriptStats = frequently() ? new ScriptStats(randomPositiveLong(), randomPositiveLong()) : null; + ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong()) : null; DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(), randomInt(), randomInt()) : null) : null; IngestStats ingestStats = null; if (frequently()) { - IngestStats.Stats totalStats = new IngestStats.Stats(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong()); + IngestStats.Stats totalStats = new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong()); int numStatsPerPipeline = randomIntBetween(0, 10); Map statsPerPipeline = new HashMap<>(); for (int i = 0; i < numStatsPerPipeline; i++) { - statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomPositiveLong(), - randomPositiveLong(), randomPositiveLong(), randomPositiveLong())); + statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomNonNegativeLong(), + randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())); } ingestStats = new IngestStats(totalStats, statsPerPipeline); } //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet - return new NodeStats(node, randomPositiveLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo, + return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo, transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, ingestStats); } } diff --git a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index be95f8dc581b9..892401356c34e 100644 --- a/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -292,7 +292,7 @@ public void testNowInScript() throws IOException { .upsert(indexRequest) .script(new Script(ScriptType.INLINE, "mock", "ctx._source.update_timestamp = ctx._now", Collections.emptyMap())) .scriptedUpsert(true); - long nowInMillis = randomPositiveLong(); + long nowInMillis = randomNonNegativeLong(); // We simulate that the document is not existing yet GetResult getResult = new GetResult("test", "type1", "2", 0, false, null, null); UpdateHelper.Result result = updateHelper.prepare(new ShardId("test", "_na_", 0), updateRequest, getResult, () -> nowInMillis); diff --git a/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java index 74427281894e9..6369ce2d6ea46 100644 --- a/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java +++ b/core/src/test/java/org/elasticsearch/common/FieldMemoryStatsTests.java @@ -95,7 +95,7 @@ public static FieldMemoryStats randomFieldMemoryStats() { ObjectLongHashMap map = new ObjectLongHashMap<>(); int keys = randomIntBetween(1, 1000); for (int i = 0; i < keys; i++) { - map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomPositiveLong()); + map.put(randomRealisticUnicodeOfCodepointLengthBetween(1, 10), randomNonNegativeLong()); } return new FieldMemoryStats(map); } diff --git a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java index 6ad5a1ce32a34..9d0a3e67e9d61 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/ByteSizeValueTests.java @@ -150,7 +150,7 @@ public void testNoDotsAllowed() { } public void testCompareEquality() { - long firstRandom = randomPositiveLong(); + long firstRandom = randomNonNegativeLong(); ByteSizeUnit randomUnit = randomFrom(ByteSizeUnit.values()); ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, randomUnit); ByteSizeValue secondByteValue = new ByteSizeValue(firstRandom, randomUnit); @@ -158,8 +158,8 @@ public void testCompareEquality() { } public void testCompareValue() { - long firstRandom = randomPositiveLong(); - long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong); + long firstRandom = randomNonNegativeLong(); + long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong); ByteSizeUnit unit = randomFrom(ByteSizeUnit.values()); ByteSizeValue firstByteValue = new ByteSizeValue(firstRandom, unit); ByteSizeValue secondByteValue = new ByteSizeValue(secondRandom, unit); @@ -168,7 +168,7 @@ public void testCompareValue() { } public void testCompareUnits() { - long number = randomPositiveLong(); + long number = randomNonNegativeLong(); ByteSizeUnit randomUnit = randomValueOtherThan(ByteSizeUnit.PB, ()->randomFrom(ByteSizeUnit.values())); ByteSizeValue firstByteValue = new ByteSizeValue(number, randomUnit); ByteSizeValue secondByteValue = new ByteSizeValue(number, ByteSizeUnit.PB); @@ -189,7 +189,7 @@ public void testConversionHashCode() { } public void testSerialization() throws IOException { - ByteSizeValue byteSizeValue = new ByteSizeValue(randomPositiveLong(), randomFrom(ByteSizeUnit.values())); + ByteSizeValue byteSizeValue = new ByteSizeValue(randomNonNegativeLong(), randomFrom(ByteSizeUnit.values())); try (BytesStreamOutput out = new BytesStreamOutput()) { byteSizeValue.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { diff --git a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java index 3a97a11308bbb..4fd2af3ce54f3 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/SizeValueTests.java @@ -66,7 +66,7 @@ public void testThatNegativeValuesThrowException() { } public void testCompareEquality() { - long randomValue = randomPositiveLong(); + long randomValue = randomNonNegativeLong(); SizeUnit randomUnit = randomFrom(SizeUnit.values()); SizeValue firstValue = new SizeValue(randomValue, randomUnit); SizeValue secondValue = new SizeValue(randomValue, randomUnit); @@ -74,8 +74,8 @@ public void testCompareEquality() { } public void testCompareValue() { - long firstRandom = randomPositiveLong(); - long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong); + long firstRandom = randomNonNegativeLong(); + long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong); SizeUnit unit = randomFrom(SizeUnit.values()); SizeValue firstSizeValue = new SizeValue(firstRandom, unit); SizeValue secondSizeValue = new SizeValue(secondRandom, unit); @@ -84,7 +84,7 @@ public void testCompareValue() { } public void testCompareUnits() { - long number = randomPositiveLong(); + long number = randomNonNegativeLong(); SizeUnit randomUnit = randomValueOtherThan(SizeUnit.PETA, ()->randomFrom(SizeUnit.values())); SizeValue firstValue = new SizeValue(number, randomUnit); SizeValue secondValue = new SizeValue(number, SizeUnit.PETA); diff --git a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java index 1f7e876f8568a..cbfd98aa3b74a 100644 --- a/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java +++ b/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java @@ -226,7 +226,7 @@ public void testToStringRep() { } public void testCompareEquality() { - long randomLong = randomPositiveLong(); + long randomLong = randomNonNegativeLong(); TimeUnit randomUnit = randomFrom(TimeUnit.values()); TimeValue firstValue = new TimeValue(randomLong, randomUnit); TimeValue secondValue = new TimeValue(randomLong, randomUnit); @@ -234,8 +234,8 @@ public void testCompareEquality() { } public void testCompareValue() { - long firstRandom = randomPositiveLong(); - long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomPositiveLong); + long firstRandom = randomNonNegativeLong(); + long secondRandom = randomValueOtherThan(firstRandom, ESTestCase::randomNonNegativeLong); TimeUnit unit = randomFrom(TimeUnit.values()); TimeValue firstValue = new TimeValue(firstRandom, unit); TimeValue secondValue = new TimeValue(secondRandom, unit); @@ -244,7 +244,7 @@ public void testCompareValue() { } public void testCompareUnits() { - long number = randomPositiveLong(); + long number = randomNonNegativeLong(); TimeUnit randomUnit = randomValueOtherThan(TimeUnit.DAYS, ()->randomFrom(TimeUnit.values())); TimeValue firstValue = new TimeValue(number, randomUnit); TimeValue secondValue = new TimeValue(number, TimeUnit.DAYS); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java index 0c4862bf32a2b..f59cc1cc6fb0f 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/ElectMasterServiceTests.java @@ -66,7 +66,8 @@ List generateRandomCandidates() { roles.add(DiscoveryNode.Role.MASTER); DiscoveryNode node = new DiscoveryNode("n_" + i, "n_" + i, buildNewFakeTransportAddress(), Collections.emptyMap(), roles, Version.CURRENT); - candidates.add(new MasterCandidate(node, randomBoolean() ? MasterCandidate.UNRECOVERED_CLUSTER_VERSION : randomPositiveLong())); + candidates.add( + new MasterCandidate(node, randomBoolean() ? MasterCandidate.UNRECOVERED_CLUSTER_VERSION : randomNonNegativeLong())); } Collections.shuffle(candidates, random()); diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index 84650f042cfb4..64d62336dd9c0 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -168,8 +168,8 @@ public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfil NetworkHandle handleD = startServices(settingsMismatch, threadPool, "UZP_D", versionD, supplier); closeables.push(handleD.transportService); - final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build(); - final ClusterState stateMismatch = ClusterState.builder(new ClusterName("mismatch")).version(randomPositiveLong()).build(); + final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build(); + final ClusterState stateMismatch = ClusterState.builder(new ClusterName("mismatch")).version(randomNonNegativeLong()).build(); Settings hostsSettings = Settings.builder() .putArray("discovery.zen.ping.unicast.hosts", @@ -329,7 +329,7 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi .put("cluster.name", "test") .build(); - final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build(); + final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build(); final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER); zenPingA.start(new PingContextProvider() { @@ -567,7 +567,7 @@ public void testResolveReuseExistingNodeConnections() throws ExecutionException, hostsSettingsBuilder.put("discovery.zen.ping.unicast.hosts", (String) null); } final Settings hostsSettings = hostsSettingsBuilder.build(); - final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build(); + final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build(); // connection to reuse handleA.transportService.connectToNode(handleB.node); @@ -639,7 +639,7 @@ public void testPingingTemporalPings() throws ExecutionException, InterruptedExc .put("cluster.name", "test") .put("discovery.zen.ping.unicast.hosts", (String) null) // use nodes for simplicity .build(); - final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomPositiveLong()).build(); + final ClusterState state = ClusterState.builder(new ClusterName("test")).version(randomNonNegativeLong()).build(); final TestUnicastZenPing zenPingA = new TestUnicastZenPing(hostsSettings, threadPool, handleA, EMPTY_HOSTS_PROVIDER); zenPingA.start(new PingContextProvider() { diff --git a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java index b58502bffe950..4d1a3ec420a46 100644 --- a/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java +++ b/core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java @@ -619,54 +619,54 @@ private FieldStats randomFieldStats(boolean withNullMinMax) throws UnknownHostEx switch (type) { case 0: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Long(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Long(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), randomLong(), randomLong()); + return new FieldStats.Long(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomLong(), randomLong()); } case 1: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Double(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Double(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), randomDouble(), randomDouble()); + return new FieldStats.Double(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), randomDouble(), randomDouble()); } case 2: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Date(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Date(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), Joda.forPattern("basicDate"), + return new FieldStats.Date(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), Joda.forPattern("basicDate"), new Date().getTime(), new Date().getTime()); } case 3: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Text(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Text(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), + return new FieldStats.Text(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), new BytesRef(randomAsciiOfLength(10)), new BytesRef(randomAsciiOfLength(20))); } case 4: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), + return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), InetAddress.getByName("::1"), InetAddress.getByName("::1")); } case 5: if (withNullMinMax && randomBoolean()) { - return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean()); + return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean()); } else { - return new FieldStats.Ip(randomPositiveLong(), randomPositiveLong(), randomPositiveLong(), - randomPositiveLong(), randomBoolean(), randomBoolean(), + return new FieldStats.Ip(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), randomBoolean(), randomBoolean(), InetAddress.getByName("1.2.3.4"), InetAddress.getByName("1.2.3.4")); } default: diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 76e1b37e4dc5b..065a6d74f07c3 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -157,6 +157,7 @@ import java.util.function.Supplier; import static java.util.Collections.emptyMap; +import static java.util.Collections.max; import static org.elasticsearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY; import static org.elasticsearch.index.engine.Engine.Operation.Origin.PRIMARY; @@ -3123,6 +3124,50 @@ public void testOutOfOrderSequenceNumbersWithVersionConflict() throws IOExceptio } } + /* + * This test tests that a no-op does not generate a new sequence number, that no-ops can advance the local checkpoint, and that no-ops + * are correctly added to the translog. + */ + public void testNoOps() throws IOException { + engine.close(); + InternalEngine noOpEngine = null; + final int maxSeqNo = randomIntBetween(0, 128); + final int localCheckpoint = randomIntBetween(0, maxSeqNo); + final int globalCheckpoint = randomIntBetween(0, localCheckpoint); + try { + final SequenceNumbersService seqNoService = + new SequenceNumbersService(shardId, defaultSettings, maxSeqNo, localCheckpoint, globalCheckpoint) { + @Override + public long generateSeqNo() { + throw new UnsupportedOperationException(); + } + }; + noOpEngine = createEngine(defaultSettings, store, primaryTranslogDir, newMergePolicy(), null, () -> seqNoService); + final long primaryTerm = randomNonNegativeLong(); + final String reason = randomAsciiOfLength(16); + noOpEngine.noOp( + new Engine.NoOp( + null, + maxSeqNo + 1, + primaryTerm, + 0, + VersionType.INTERNAL, + randomFrom(PRIMARY, REPLICA, PEER_RECOVERY, LOCAL_TRANSLOG_RECOVERY), + System.nanoTime(), + reason)); + assertThat(noOpEngine.seqNoService().getLocalCheckpoint(), equalTo((long) (maxSeqNo + 1))); + assertThat(noOpEngine.getTranslog().totalOperations(), equalTo(1)); + final Translog.Operation op = noOpEngine.getTranslog().newSnapshot().next(); + assertThat(op, instanceOf(Translog.NoOp.class)); + final Translog.NoOp noOp = (Translog.NoOp) op; + assertThat(noOp.seqNo(), equalTo((long) (maxSeqNo + 1))); + assertThat(noOp.primaryTerm(), equalTo(primaryTerm)); + assertThat(noOp.reason(), equalTo(reason)); + } finally { + IOUtils.close(noOpEngine); + } + } + /** * Return a tuple representing the sequence ID for the given {@code Get} * operation. The first value in the tuple is the sequence number, the diff --git a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java index 54881bad88485..90995bec34d34 100644 --- a/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java +++ b/core/src/test/java/org/elasticsearch/index/fielddata/FieldDataStatsTests.java @@ -30,7 +30,7 @@ public class FieldDataStatsTests extends ESTestCase { public void testSerialize() throws IOException { FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats(); - FieldDataStats stats = new FieldDataStats(randomPositiveLong(), randomPositiveLong(), map == null ? null : + FieldDataStats stats = new FieldDataStats(randomNonNegativeLong(), randomNonNegativeLong(), map == null ? null : map); BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); diff --git a/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java index d82c42214043b..a40644ea2cbee 100644 --- a/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java +++ b/core/src/test/java/org/elasticsearch/index/get/GetResultTests.java @@ -96,7 +96,7 @@ public static GetResult mutateGetResult(GetResult getResult) { getResult.isExists(), getResult.internalSourceRef(), getResult.getFields())); mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), randomUnicodeOfLength(15), getResult.getVersion(), getResult.isExists(), getResult.internalSourceRef(), getResult.getFields())); - mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), randomPositiveLong(), + mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), randomNonNegativeLong(), getResult.isExists(), getResult.internalSourceRef(), getResult.getFields())); mutations.add(() -> new GetResult(getResult.getIndex(), getResult.getType(), getResult.getId(), getResult.getVersion(), getResult.isExists() == false, getResult.internalSourceRef(), getResult.getFields())); @@ -117,7 +117,7 @@ public static Tuple randomGetResult(XContentType xContentT Map fields = null; Map expectedFields = null; if (frequently()) { - version = randomPositiveLong(); + version = randomNonNegativeLong(); exists = true; if (frequently()) { source = RandomObjects.randomSource(random()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 15f1819e939a9..c714a0f280d5d 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -67,7 +67,7 @@ public void modify(MappedFieldType ft) { ((DateFieldType) ft).setDateTimeFormatter(Joda.forPattern("date_optional_time", Locale.CANADA)); } }); - nowInMillis = randomPositiveLong(); + nowInMillis = randomNonNegativeLong(); } public void testIsFieldWithinQueryEmptyReader() throws IOException { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 04f5dcb5b912c..f70db120fcfc3 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -48,7 +48,7 @@ public class RangeFieldTypeTests extends FieldTypeTestCase { @Before public void setupProperties() { type = RandomPicks.randomFrom(random(), RangeType.values()); - nowInMillis = randomPositiveLong(); + nowInMillis = randomNonNegativeLong(); if (type == RangeType.DATE) { addModifier(new Modifier("format", true) { @Override diff --git a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 434c3d1ebf3f8..b83c1b2897a3c 100644 --- a/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -46,7 +46,7 @@ public void testFailIfFieldMappingNotFound() { IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY); MapperService mapperService = mock(MapperService.class); when(mapperService.getIndexSettings()).thenReturn(indexSettings); - final long nowInMillis = randomPositiveLong(); + final long nowInMillis = randomNonNegativeLong(); QueryShardContext context = new QueryShardContext( 0, indexSettings, null, null, mapperService, null, null, xContentRegistry(), null, null, () -> nowInMillis); diff --git a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java index 510371c2d51b8..3f8b3a1d9e4d3 100644 --- a/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java +++ b/core/src/test/java/org/elasticsearch/index/seqno/GlobalCheckpointSyncActionTests.java @@ -36,10 +36,7 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collections; -import java.util.HashSet; import static org.elasticsearch.mock.orig.Mockito.when; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; @@ -103,7 +100,8 @@ public void testTranslogSyncAfterGlobalCheckpointSync() throws Exception { if (randomBoolean()) { action.shardOperationOnPrimary(primaryRequest, indexShard); } else { - action.shardOperationOnReplica(new GlobalCheckpointSyncAction.ReplicaRequest(primaryRequest, randomPositiveLong()), indexShard); + action.shardOperationOnReplica( + new GlobalCheckpointSyncAction.ReplicaRequest(primaryRequest, randomNonNegativeLong()), indexShard); } verify(translog).sync(); diff --git a/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java index 71ffffdcbff6e..2d699b2c4571c 100644 --- a/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java +++ b/core/src/test/java/org/elasticsearch/index/suggest/stats/CompletionsStatsTests.java @@ -31,7 +31,7 @@ public class CompletionsStatsTests extends ESTestCase { public void testSerialize() throws IOException { FieldMemoryStats map = randomBoolean() ? null : FieldMemoryStatsTests.randomFieldMemoryStats(); - CompletionStats stats = new CompletionStats(randomPositiveLong(), map == null ? null : + CompletionStats stats = new CompletionStats(randomNonNegativeLong(), map == null ? null : map); BytesStreamOutput out = new BytesStreamOutput(); stats.writeTo(out); diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index 41118d6efa2e2..a2a6620a6c242 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -31,7 +31,6 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -88,9 +87,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.hasToString; @LuceneTestCase.SuppressFileSystems("ExtrasFS") public class TranslogTests extends ESTestCase { @@ -226,17 +227,28 @@ public void testSimpleOperations() throws IOException { assertThat(snapshot, SnapshotMatchers.equalsTo(ops)); assertThat(snapshot.totalOperations(), equalTo(ops.size())); + final long seqNo = randomNonNegativeLong(); + final long primaryTerm = randomNonNegativeLong(); + final String reason = randomAsciiOfLength(16); + addToTranslogAndList(translog, ops, new Translog.NoOp(seqNo, primaryTerm, reason)); + snapshot = translog.newSnapshot(); Translog.Index index = (Translog.Index) snapshot.next(); - assertThat(index != null, equalTo(true)); + assertNotNull(index); assertThat(BytesReference.toBytes(index.source()), equalTo(new byte[]{1})); Translog.Delete delete = (Translog.Delete) snapshot.next(); - assertThat(delete != null, equalTo(true)); + assertNotNull(delete); assertThat(delete.uid(), equalTo(newUid("2"))); - assertThat(snapshot.next(), equalTo(null)); + Translog.NoOp noOp = (Translog.NoOp) snapshot.next(); + assertNotNull(noOp); + assertThat(noOp.seqNo(), equalTo(seqNo)); + assertThat(noOp.primaryTerm(), equalTo(primaryTerm)); + assertThat(noOp.reason(), equalTo(reason)); + + assertNull(snapshot.next()); long firstId = translog.currentFileGeneration(); translog.prepareCommit(); @@ -268,70 +280,106 @@ protected TranslogStats stats() throws IOException { public void testStats() throws IOException { final long firstOperationPosition = translog.getFirstOperationPosition(); - TranslogStats stats = stats(); - assertThat(stats.estimatedNumberOfOperations(), equalTo(0L)); - long lastSize = stats.getTranslogSizeInBytes(); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(0L)); + } assertThat((int) firstOperationPosition, greaterThan(CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC))); - assertThat(lastSize, equalTo(firstOperationPosition)); - TranslogStats total = new TranslogStats(); translog.add(new Translog.Index("test", "1", new byte[]{1})); - stats = stats(); - total.add(stats); - assertThat(stats.estimatedNumberOfOperations(), equalTo(1L)); - assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize)); - lastSize = stats.getTranslogSizeInBytes(); + + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(1L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(97L)); + } translog.add(new Translog.Delete(newUid("2"))); - stats = stats(); - total.add(stats); - assertThat(stats.estimatedNumberOfOperations(), equalTo(2L)); - assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize)); - lastSize = stats.getTranslogSizeInBytes(); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(2L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(125L)); + } translog.add(new Translog.Delete(newUid("3"))); - translog.prepareCommit(); - stats = stats(); - total.add(stats); - assertThat(stats.estimatedNumberOfOperations(), equalTo(3L)); - assertThat(stats.getTranslogSizeInBytes(), greaterThan(lastSize)); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(3L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(153L)); + } - translog.commit(); - stats = stats(); - total.add(stats); - assertThat(stats.estimatedNumberOfOperations(), equalTo(0L)); - assertThat(stats.getTranslogSizeInBytes(), equalTo(firstOperationPosition)); - assertEquals(6, total.estimatedNumberOfOperations()); - assertEquals(419, total.getTranslogSizeInBytes()); + final long seqNo = 1; + final long primaryTerm = 1; + translog.add(new Translog.NoOp(seqNo, primaryTerm, randomAsciiOfLength(16))); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(4L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(195L)); + } - BytesStreamOutput out = new BytesStreamOutput(); - total.writeTo(out); - TranslogStats copy = new TranslogStats(); - copy.readFrom(out.bytes().streamInput()); + final long expectedSizeInBytes = 238L; + translog.prepareCommit(); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(4L)); + assertThat( + stats.getTranslogSizeInBytes(), + equalTo(expectedSizeInBytes)); + } - assertEquals(6, copy.estimatedNumberOfOperations()); - assertEquals(419, copy.getTranslogSizeInBytes()); + { + final TranslogStats stats = stats(); + final BytesStreamOutput out = new BytesStreamOutput(); + stats.writeTo(out); + final TranslogStats copy = new TranslogStats(); + copy.readFrom(out.bytes().streamInput()); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - builder.startObject(); - copy.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - assertEquals("{\"translog\":{\"operations\":6,\"size_in_bytes\":419}}", builder.string()); - } + assertThat(copy.estimatedNumberOfOperations(), equalTo(4L)); + assertThat(copy.getTranslogSizeInBytes(), equalTo(expectedSizeInBytes)); - try { - new TranslogStats(1, -1); - fail("must be positive"); - } catch (IllegalArgumentException ex) { - //all well + try (final XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + copy.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertThat(builder.string(), equalTo("{\"translog\":{\"operations\":4,\"size_in_bytes\":" + expectedSizeInBytes + "}}")); + } } - try { - new TranslogStats(-1, 1); - fail("must be positive"); - } catch (IllegalArgumentException ex) { - //all well + + translog.commit(); + { + final TranslogStats stats = stats(); + assertThat(stats.estimatedNumberOfOperations(), equalTo(0L)); + assertThat(stats.getTranslogSizeInBytes(), equalTo(firstOperationPosition)); } } + public void testTotalTests() { + final TranslogStats total = new TranslogStats(); + final int n = randomIntBetween(0, 16); + final List statsList = new ArrayList<>(n); + for (int i = 0; i < n; i++) { + final TranslogStats stats = new TranslogStats(randomIntBetween(1, 4096), randomIntBetween(1, 1 << 20)); + statsList.add(stats); + total.add(stats); + } + + assertThat( + total.estimatedNumberOfOperations(), + equalTo(statsList.stream().mapToLong(TranslogStats::estimatedNumberOfOperations).sum())); + assertThat( + total.getTranslogSizeInBytes(), + equalTo(statsList.stream().mapToLong(TranslogStats::getTranslogSizeInBytes).sum())); + } + + public void testNegativeNumberOfOperations() { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new TranslogStats(-1, 1)); + assertThat(e, hasToString(containsString("numberOfOperations must be >= 0"))); + } + + public void testNegativeSizeInBytes() { + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new TranslogStats(1, -1)); + assertThat(e, hasToString(containsString("translogSizeInBytes must be >= 0"))); + } + public void testSnapshot() throws IOException { ArrayList ops = new ArrayList<>(); Translog.Snapshot snapshot = translog.newSnapshot(); @@ -465,8 +513,15 @@ public void testConcurrentWritesWithVaryingSize() throws Throwable { assertEquals(expDelOp.version(), delOp.version()); assertEquals(expDelOp.versionType(), delOp.versionType()); break; + case NO_OP: + final Translog.NoOp noOp = (Translog.NoOp) op; + final Translog.NoOp expectedNoOp = (Translog.NoOp) expectedOp; + assertThat(noOp.seqNo(), equalTo(expectedNoOp.seqNo())); + assertThat(noOp.primaryTerm(), equalTo(expectedNoOp.primaryTerm())); + assertThat(noOp.reason(), equalTo(expectedNoOp.reason())); + break; default: - throw new ElasticsearchException("unsupported opType"); + throw new AssertionError("unsupported operation type [" + op.opType() + "]"); } } @@ -607,7 +662,9 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep while (run.get()) { long id = idGenerator.incrementAndGet(); final Translog.Operation op; - switch (Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))]) { + final Translog.Operation.Type type = + Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))]; + switch (type) { case CREATE: case INDEX: op = new Translog.Index("type", "" + id, new byte[]{(byte) id}); @@ -615,8 +672,11 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep case DELETE: op = new Translog.Delete(newUid("" + id)); break; + case NO_OP: + op = new Translog.NoOp(id, id, Long.toString(id)); + break; default: - throw new ElasticsearchException("unknown type"); + throw new AssertionError("unsupported operation type [" + type + "]"); } Translog.Location location = translog.add(op); Translog.Location existing = writtenOps.put(op, location); @@ -1137,7 +1197,7 @@ public void testRecoveryUncommittedCorruptedCheckpoint() throws IOException { try (Translog ignored = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) { fail("corrupted"); } catch (IllegalStateException ex) { - assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=2353, numOps=55, translogFileGeneration=2, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration=0, globalCheckpoint=-2}"); + assertEquals(ex.getMessage(), "Checkpoint file translog-2.ckp already exists but has corrupted content expected: Checkpoint{offset=3123, numOps=55, translogFileGeneration=2, globalCheckpoint=-2} but got: Checkpoint{offset=0, numOps=0, translogFileGeneration=0, globalCheckpoint=-2}"); } Checkpoint.write(FileChannel::open, config.getTranslogPath().resolve(Translog.getCommitCheckpointFileName(read.generation)), read, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); try (Translog translog = new Translog(config, translogGeneration, () -> SequenceNumbersService.UNASSIGNED_SEQ_NO)) { @@ -1293,7 +1353,8 @@ public void run() { downLatch.await(); for (int opCount = 0; opCount < opsPerThread; opCount++) { Translog.Operation op; - switch (randomFrom(Translog.Operation.Type.values())) { + final Translog.Operation.Type type = randomFrom(Translog.Operation.Type.values()); + switch (type) { case CREATE: case INDEX: op = new Translog.Index("test", threadId + "_" + opCount, @@ -1307,8 +1368,11 @@ public void run() { 1 + randomInt(100000), randomFrom(VersionType.values())); break; + case NO_OP: + op = new Translog.NoOp(randomNonNegativeLong(), randomNonNegativeLong(), randomAsciiOfLength(16)); + break; default: - throw new ElasticsearchException("not supported op type"); + throw new AssertionError("unsupported operation type [" + type + "]"); } Translog.Location loc = add(op); diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java index 9bd4ab3b90f33..5d817b6321d5b 100644 --- a/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java @@ -41,7 +41,7 @@ public class OsProbeTests extends ESTestCase { public void testOsInfo() { int allocatedProcessors = randomIntBetween(1, Runtime.getRuntime().availableProcessors()); - long refreshInterval = randomBoolean() ? -1 : randomPositiveLong(); + long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong(); OsInfo info = probe.osInfo(refreshInterval, allocatedProcessors); assertNotNull(info); assertEquals(refreshInterval, info.getRefreshInterval()); diff --git a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java index 8334f71e86a67..9f8b60a55a9d7 100644 --- a/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/os/OsStatsTests.java @@ -38,11 +38,11 @@ public void testSerialization() throws IOException { OsStats.Swap swap = new OsStats.Swap(randomLong(), randomLong()); OsStats.Cgroup cgroup = new OsStats.Cgroup( randomAsciiOfLength(8), - randomPositiveLong(), + randomNonNegativeLong(), randomAsciiOfLength(8), - randomPositiveLong(), - randomPositiveLong(), - new OsStats.Cgroup.CpuStat(randomPositiveLong(), randomPositiveLong(), randomPositiveLong())); + randomNonNegativeLong(), + randomNonNegativeLong(), + new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())); OsStats osStats = new OsStats(System.currentTimeMillis(), cpu, mem, swap, cgroup); try (BytesStreamOutput out = new BytesStreamOutput()) { diff --git a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java index f8fc6a675cfd8..4a44c518051a2 100644 --- a/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java +++ b/core/src/test/java/org/elasticsearch/monitor/process/ProcessProbeTests.java @@ -36,7 +36,7 @@ public class ProcessProbeTests extends ESTestCase { private final ProcessProbe probe = ProcessProbe.getInstance(); public void testProcessInfo() { - long refreshInterval = randomPositiveLong(); + long refreshInterval = randomNonNegativeLong(); ProcessInfo info = probe.processInfo(refreshInterval); assertNotNull(info); assertEquals(refreshInterval, info.getRefreshInterval()); diff --git a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java index 59077367904bf..9c36a7a649ad6 100644 --- a/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java +++ b/core/src/test/java/org/elasticsearch/nodesinfo/NodeInfoStreamingTests.java @@ -112,13 +112,13 @@ private static NodeInfo createNodeInfo() { if (randomBoolean()) { int availableProcessors = randomIntBetween(1, 64); int allocatedProcessors = randomIntBetween(1, availableProcessors); - long refreshInterval = randomBoolean() ? -1 : randomPositiveLong(); + long refreshInterval = randomBoolean() ? -1 : randomNonNegativeLong(); String name = randomAsciiOfLengthBetween(3, 10); String arch = randomAsciiOfLengthBetween(3, 10); String version = randomAsciiOfLengthBetween(3, 10); osInfo = new OsInfo(refreshInterval, availableProcessors, allocatedProcessors, name, arch, version); } - ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomPositiveLong()); + ProcessInfo process = randomBoolean() ? null : new ProcessInfo(randomInt(), randomBoolean(), randomNonNegativeLong()); JvmInfo jvm = randomBoolean() ? null : JvmInfo.jvmInfo(); ThreadPoolInfo threadPoolInfo = null; if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java index 031cab1286a7a..9e58bf26744d0 100644 --- a/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java +++ b/core/src/test/java/org/elasticsearch/search/SearchRequestTests.java @@ -95,7 +95,7 @@ private SearchRequest mutate(SearchRequest searchRequest) throws IOException { mutators.add(() -> mutation.routing(randomValueOtherThan(searchRequest.routing(), () -> randomAsciiOfLengthBetween(3, 10)))); mutators.add(() -> mutation.requestCache((randomValueOtherThan(searchRequest.requestCache(), () -> randomBoolean())))); mutators.add(() -> mutation - .scroll(randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomPositiveLong() % 100000))))); + .scroll(randomValueOtherThan(searchRequest.scroll(), () -> new Scroll(new TimeValue(randomNonNegativeLong() % 100000))))); mutators.add(() -> mutation.searchType(randomValueOtherThan(searchRequest.searchType(), () -> randomFrom(SearchType.values())))); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); randomFrom(mutators).run(); diff --git a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java index 1ac267330d77f..a53a41f4c1885 100644 --- a/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java +++ b/core/src/test/java/org/elasticsearch/search/internal/ShardSearchTransportRequestTests.java @@ -198,7 +198,7 @@ public void testSerialize50Request() throws IOException { .putAlias(AliasMetaData.newAliasMetaDataBuilder("UjLlLkjwWh").filter("{\"term\" : {\"foo\" : \"bar1\"}}")) .putAlias(AliasMetaData.newAliasMetaDataBuilder("uBpgtwuqDG").filter("{\"term\" : {\"foo\" : \"bar2\"}}")); IndexSettings indexSettings = new IndexSettings(indexMetadata.build(), Settings.EMPTY); - final long nowInMillis = randomPositiveLong(); + final long nowInMillis = randomNonNegativeLong(); QueryShardContext context = new QueryShardContext( 0, indexSettings, null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis); readRequest.rewrite(context); diff --git a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java index 3e051c9828b32..d4429a5502285 100644 --- a/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/rescore/QueryRescoreBuilderTests.java @@ -133,7 +133,7 @@ public void testFromXContent() throws IOException { * than the test builder */ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, IOException { - final long nowInMillis = randomPositiveLong(); + final long nowInMillis = randomNonNegativeLong(); Settings indexSettings = Settings.builder() .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAsciiOfLengthBetween(1, 10), indexSettings); diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 683d5d78c9e0f..fda35d8bd7c0f 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -208,7 +208,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { public void onCache(ShardId shardId, Accountable accountable) { } }); - long nowInMillis = randomPositiveLong(); + long nowInMillis = randomNonNegativeLong(); return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService, xContentRegistry(), null, null, () -> nowInMillis) { @Override diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java index 7c4b4e3274fab..b2e92b15116f2 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RoundTripTests.java @@ -217,7 +217,7 @@ public void testBulkByTaskStatus() throws IOException { } public void testReindexResponse() throws IOException { - BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(), + BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomNonNegativeLong()), randomStatus(), randomIndexingFailures(), randomSearchFailures(), randomBoolean()); BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse(); roundTrip(response, tripped); @@ -225,7 +225,7 @@ public void testReindexResponse() throws IOException { } public void testBulkIndexByScrollResponse() throws IOException { - BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomPositiveLong()), randomStatus(), + BulkIndexByScrollResponse response = new BulkIndexByScrollResponse(timeValueMillis(randomNonNegativeLong()), randomStatus(), randomIndexingFailures(), randomSearchFailures(), randomBoolean()); BulkIndexByScrollResponse tripped = new BulkIndexByScrollResponse(); roundTrip(response, tripped); diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java index ca0b5964dc202..0c80bcc71fd15 100644 --- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java +++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java @@ -60,7 +60,7 @@ public static void loadDatabaseReaders() throws IOException { Files.copy(new ByteArrayInputStream(StreamsUtils.copyToBytesFromClasspath("/GeoLite2-Country.mmdb.gz")), geoIpConfigDir.resolve("GeoLite2-Country.mmdb.gz")); - NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomPositiveLong())); + NodeCache cache = randomFrom(NoCache.getInstance(), new GeoIpCache(randomNonNegativeLong())); databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpConfigDir, cache); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 243e328e5dffe..468a5aaf3a433 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -1023,7 +1023,7 @@ private static class ServiceHolder implements Closeable { private final BitsetFilterCache bitsetFilterCache; private final ScriptService scriptService; private final Client client; - private final long nowInMillis = randomPositiveLong(); + private final long nowInMillis = randomNonNegativeLong(); ServiceHolder(Settings nodeSettings, Settings indexSettings, Collection> plugins, AbstractQueryTestCase testCase) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index bcfad66ef59d2..567c987ebf02a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -441,7 +441,7 @@ public static int randomInt() { return random().nextInt(); } - public static long randomPositiveLong() { + public static long randomNonNegativeLong() { long randomLong; do { randomLong = randomLong(); diff --git a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java index 2542e1c374b5b..61f7722a7c201 100644 --- a/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java +++ b/test/framework/src/test/java/org/elasticsearch/search/MockSearchServiceTests.java @@ -33,7 +33,7 @@ public class MockSearchServiceTests extends ESTestCase { public void testAssertNoInFlightContext() { - final long nowInMillis = randomPositiveLong(); + final long nowInMillis = randomNonNegativeLong(); SearchContext s = new TestSearchContext(new QueryShardContext(0, new IndexSettings(IndexMetaData.PROTO, Settings.EMPTY), null, null, null, null, null, xContentRegistry(), null, null, () -> nowInMillis)) { @Override From 06576ed13b038da71a23d0cd01b49fc1fd240625 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 22 Dec 2016 10:49:18 +0000 Subject: [PATCH 09/22] Adds abstract test classes for serialisation (#22281) This adds test classes that can be used to test the wire serialisation and (optionally) the XContent serialisation of objects that implement Streamable/Writeable and ToXContent. These test classes will enable classes sich as InternalAggregation (or at least its implementations) to be tested in a consistent way when is comes to testing serialisation. --- .../aggregations/InternalAggregation.java | 43 ++++++ .../InternalNumericMetricsAggregation.java | 13 ++ .../aggregations/metrics/min/InternalMin.java | 12 ++ .../metrics/min/InternalMinTests.java | 62 +++++++++ .../test/AbstractSerializingTestCase.java | 104 ++++++++++++++ .../test/AbstractStreamableTestCase.java | 127 ++++++++++++++++++ .../AbstractStreamableXContentTestCase.java | 102 ++++++++++++++ .../test/AbstractWireSerializingTestCase.java | 123 +++++++++++++++++ 8 files changed, 586 insertions(+) create mode 100644 core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java index df25f0e263551..3b73d8b13d70c 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; /** * An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations. @@ -189,6 +190,48 @@ public final XContentBuilder toXContent(XContentBuilder builder, Params params) public abstract XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException; + @Override + public int hashCode() { + return Objects.hash(name, metaData, pipelineAggregators, doHashCode()); + } + + // norelease: make this abstract when all InternalAggregations implement this method + /** + * Opportunity for subclasses to the {@link #hashCode(Object)} for this + * class. + **/ + protected int doHashCode() { + return System.identityHashCode(this); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + InternalAggregation other = (InternalAggregation) obj; + return Objects.equals(name, other.name) && + Objects.equals(pipelineAggregators, other.pipelineAggregators) && + Objects.equals(metaData, other.metaData) && + doEquals(obj); + } + + // norelease: make this abstract when all InternalAggregations implement this method + /** + * Opportunity for subclasses to add criteria to the {@link #equals(Object)} + * method for this class. + * + * This method can safely cast obj to the subclass since the + * {@link #equals(Object)} method checks that obj is the same + * class as this + */ + protected boolean doEquals(Object obj) { + return this == obj; + } + /** * Common xcontent fields that are shared among addAggregation */ diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java index 9f591c7d4252f..b9442637a6bb9 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; public abstract class InternalNumericMetricsAggregation extends InternalMetricsAggregation { @@ -102,4 +103,16 @@ private InternalNumericMetricsAggregation(String name, List protected InternalNumericMetricsAggregation(StreamInput in) throws IOException { super(in); } + + @Override + protected int doHashCode() { + return Objects.hash(format, super.hashCode()); + } + + @Override + protected boolean doEquals(Object obj) { + InternalNumericMetricsAggregation other = (InternalNumericMetricsAggregation) obj; + return super.equals(obj) && + Objects.equals(format, other.format); + } } diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java index 09f9c3915fdfd..46481d1837f6b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/min/InternalMin.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; public class InternalMin extends InternalNumericMetricsAggregation.SingleValue implements Min { private final double min; @@ -89,4 +90,15 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th return builder; } + @Override + protected int doHashCode() { + return Objects.hash(min); + } + + @Override + protected boolean doEquals(Object obj) { + InternalMin other = (InternalMin) obj; + return Objects.equals(min, other.min); + } + } diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java new file mode 100644 index 0000000000000..dae49ff7fc382 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/search/aggregations/metrics/min/InternalMinTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.metrics.min; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class InternalMinTests extends AbstractWireSerializingTestCase { + + @Override + protected InternalMin createTestInstance() { + return new InternalMin(randomAsciiOfLengthBetween(1, 20), randomDouble(), + randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.GEOHASH, DocValueFormat.IP, DocValueFormat.RAW), Collections.emptyList(), + new HashMap<>()); + } + + @Override + protected Reader instanceReader() { + return InternalMin::new; + } + + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(); + entries.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.BOOLEAN.getWriteableName(), + in -> DocValueFormat.BOOLEAN)); + entries.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new)); + entries.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new)); + entries.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), + in -> DocValueFormat.GEOHASH)); + entries.add(new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP)); + entries.add( + new NamedWriteableRegistry.Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW)); + return new NamedWriteableRegistry(entries); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java new file mode 100644 index 0000000000000..8d5556eeabb82 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractSerializingTestCase.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +public abstract class AbstractSerializingTestCase extends AbstractWireSerializingTestCase { + + /** + * Generic test that creates new instance from the test instance and checks + * both for equality and asserts equality on the two instances. + */ + public void testFromXContent() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = toXContent(testInstance, xContentType); + XContentBuilder shuffled = shuffleXContent(builder); + assertParsedInstance(xContentType, shuffled.bytes(), testInstance); + for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { + String instanceAsString = alternateVersion.getKey(); + assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue()); + } + } + } + + private void assertParsedInstance(XContentType xContentType, BytesReference instanceAsBytes, T expectedInstance) + throws IOException { + + XContentParser parser = createParser(XContentFactory.xContent(xContentType), instanceAsBytes); + T newInstance = parseInstance(parser); + assertNotSame(newInstance, expectedInstance); + assertEquals(expectedInstance, newInstance); + assertEquals(expectedInstance.hashCode(), newInstance.hashCode()); + } + + private T parseInstance(XContentParser parser) throws IOException { + T parsedInstance = doParseInstance(parser); + assertNull(parser.nextToken()); + return parsedInstance; + } + + /** + * Parses to a new instance using the provided {@link XContentParser} + */ + protected abstract T doParseInstance(XContentParser parser); + + /** + * Renders the provided instance in XContent + * + * @param instance + * the instance to render + * @param contentType + * the content type to render to + */ + protected XContentBuilder toXContent(T instance, XContentType contentType) + throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + instance.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + /** + * Returns alternate string representation of the instance that need to be + * tested as they are never used as output of the test instance. By default + * there are no alternate versions. + * + * These alternatives must be JSON strings. + */ + protected Map getAlternateVersions() { + return Collections.emptyMap(); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java new file mode 100644 index 0000000000000..0a6d7eca270ea --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableTestCase.java @@ -0,0 +1,127 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Streamable; + +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractStreamableTestCase extends ESTestCase { + protected static final int NUMBER_OF_TEST_RUNS = 20; + + /** + * Creates a random test instance to use in the tests. This method will be + * called multiple times during test execution and should return a different + * random instance each time it is called. + */ + protected abstract T createTestInstance(); + + /** + * Creates an empty instance to use when deserialising the + * {@link Streamable}. This usually returns an instance created using the + * zer-arg constructor + */ + protected abstract T createBlankInstance(); + + /** + * Tests that the equals and hashcode methods are consistent and copied + * versions of the instance have are equal. + */ + public void testEqualsAndHashcode() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T firstInstance = createTestInstance(); + assertFalse("instance is equal to null", firstInstance.equals(null)); + assertFalse("instance is equal to incompatible type", firstInstance.equals("")); + assertEquals("instance is not equal to self", firstInstance, firstInstance); + assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(), + equalTo(firstInstance.hashCode())); + + T secondInstance = copyInstance(firstInstance); + assertEquals("instance is not equal to self", secondInstance, secondInstance); + assertEquals("instance is not equal to its copy", firstInstance, secondInstance); + assertEquals("equals is not symmetric", secondInstance, firstInstance); + assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(), + equalTo(firstInstance.hashCode())); + + T thirdInstance = copyInstance(secondInstance); + assertEquals("instance is not equal to self", thirdInstance, thirdInstance); + assertEquals("instance is not equal to its copy", secondInstance, thirdInstance); + assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(), + equalTo(thirdInstance.hashCode())); + assertEquals("equals is not transitive", firstInstance, thirdInstance); + assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(), + equalTo(thirdInstance.hashCode())); + assertEquals("equals is not symmetric", thirdInstance, secondInstance); + assertEquals("equals is not symmetric", thirdInstance, firstInstance); + } + } + + /** + * Test serialization and deserialization of the test instance. + */ + public void testSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + assertSerialization(testInstance); + } + } + + /** + * Serialize the given instance and asserts that both are equal + */ + protected T assertSerialization(T testInstance) throws IOException { + T deserializedInstance = copyInstance(testInstance); + assertEquals(testInstance, deserializedInstance); + assertEquals(testInstance.hashCode(), deserializedInstance.hashCode()); + assertNotSame(testInstance, deserializedInstance); + return deserializedInstance; + } + + private T copyInstance(T instance) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + instance.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), + getNamedWriteableRegistry())) { + T newInstance = createBlankInstance(); + newInstance.readFrom(in); + return newInstance; + } + } + } + + /** + * Get the {@link NamedWriteableRegistry} to use when de-serializing the object. + * + * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize. + * + * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s + */ + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.emptyList()); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java new file mode 100644 index 0000000000000..79bd9d09223f4 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractStreamableXContentTestCase.java @@ -0,0 +1,102 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +public abstract class AbstractStreamableXContentTestCase extends AbstractStreamableTestCase { + + /** + * Generic test that creates new instance from the test instance and checks + * both for equality and asserts equality on the two queries. + */ + public void testFromXContent() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + XContentType xContentType = randomFrom(XContentType.values()); + XContentBuilder builder = toXContent(testInstance, xContentType); + XContentBuilder shuffled = shuffleXContent(builder); + assertParsedInstance(xContentType, shuffled.bytes(), testInstance); + for (Map.Entry alternateVersion : getAlternateVersions().entrySet()) { + String instanceAsString = alternateVersion.getKey(); + assertParsedInstance(XContentType.JSON, new BytesArray(instanceAsString), alternateVersion.getValue()); + } + } + } + + private void assertParsedInstance(XContentType xContentType, BytesReference instanceAsBytes, T expectedInstance) + throws IOException { + XContentParser parser = createParser(XContentFactory.xContent(xContentType), instanceAsBytes); + T newInstance = parseInstance(parser); + assertNotSame(newInstance, expectedInstance); + assertEquals(expectedInstance, newInstance); + assertEquals(expectedInstance.hashCode(), newInstance.hashCode()); + } + + private T parseInstance(XContentParser parser) throws IOException { + T parsedInstance = doParseInstance(parser); + assertNull(parser.nextToken()); + return parsedInstance; + } + + /** + * Parses to a new instance using the provided {@link XContentParser} + */ + protected abstract T doParseInstance(XContentParser parser); + + /** + * Renders the provided instance in XContent + * + * @param instance + * the instance to render + * @param contentType + * the content type to render to + */ + protected static XContentBuilder toXContent(T instance, XContentType contentType) + throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + instance.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + /** + * Returns alternate string representation of the instance that need to be + * tested as they are never used as output of the test instance. By default + * there are no alternate versions. + * + * These alternatives must be JSON strings. + */ + protected Map getAlternateVersions() { + return Collections.emptyMap(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java new file mode 100644 index 0000000000000..bf65a7f4bdd4a --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractWireSerializingTestCase.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.test; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.io.stream.Writeable.Reader; + +import java.io.IOException; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractWireSerializingTestCase extends ESTestCase { + protected static final int NUMBER_OF_TEST_RUNS = 20; + + /** + * Creates a random test instance to use in the tests. This method will be + * called multiple times during test execution and should return a different + * random instance each time it is called. + */ + protected abstract T createTestInstance(); + + /** + * Returns a {@link Reader} that can be used to de-serialize the instance + */ + protected abstract Reader instanceReader(); + + /** + * Tests that the equals and hashcode methods are consistent and copied + * versions of the instance have are equal. + */ + public void testEqualsAndHashcode() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T firstInstance = createTestInstance(); + assertFalse("instance is equal to null", firstInstance.equals(null)); + assertFalse("instance is equal to incompatible type", firstInstance.equals("")); + assertEquals("instance is not equal to self", firstInstance, firstInstance); + assertThat("same instance's hashcode returns different values if called multiple times", firstInstance.hashCode(), + equalTo(firstInstance.hashCode())); + + T secondInstance = copyInstance(firstInstance); + assertEquals("instance is not equal to self", secondInstance, secondInstance); + assertEquals("instance is not equal to its copy", firstInstance, secondInstance); + assertEquals("equals is not symmetric", secondInstance, firstInstance); + assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(), + equalTo(firstInstance.hashCode())); + + T thirdInstance = copyInstance(secondInstance); + assertEquals("instance is not equal to self", thirdInstance, thirdInstance); + assertEquals("instance is not equal to its copy", secondInstance, thirdInstance); + assertThat("instance copy's hashcode is different from original hashcode", secondInstance.hashCode(), + equalTo(thirdInstance.hashCode())); + assertEquals("equals is not transitive", firstInstance, thirdInstance); + assertThat("instance copy's hashcode is different from original hashcode", firstInstance.hashCode(), + equalTo(thirdInstance.hashCode())); + assertEquals("equals is not symmetric", thirdInstance, secondInstance); + assertEquals("equals is not symmetric", thirdInstance, firstInstance); + } + } + + /** + * Test serialization and deserialization of the test instance. + */ + public void testSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + assertSerialization(testInstance); + } + } + + /** + * Serialize the given instance and asserts that both are equal + */ + protected T assertSerialization(T testInstance) throws IOException { + T deserializedInstance = copyInstance(testInstance); + assertEquals(testInstance, deserializedInstance); + assertEquals(testInstance.hashCode(), deserializedInstance.hashCode()); + assertNotSame(testInstance, deserializedInstance); + return deserializedInstance; + } + + private T copyInstance(T instance) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + instance.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), + getNamedWriteableRegistry())) { + return instanceReader().read(in); + } + } + } + + /** + * Get the {@link NamedWriteableRegistry} to use when de-serializing the object. + * + * Override this method if you need to register {@link NamedWriteable}s for the test object to de-serialize. + * + * By default this will return a {@link NamedWriteableRegistry} with no registered {@link NamedWriteable}s + */ + protected NamedWriteableRegistry getNamedWriteableRegistry() { + return new NamedWriteableRegistry(Collections.emptyList()); + } +} From acd64c6ee1518d45fac3362ce753bc0801f439a2 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 22 Dec 2016 14:18:30 +0100 Subject: [PATCH 10/22] fixed jdocs and removed already fixed norelease --- .../search/aggregations/InternalAggregation.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java index 3b73d8b13d70c..9928d7eb127be 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/InternalAggregation.java @@ -195,9 +195,8 @@ public int hashCode() { return Objects.hash(name, metaData, pipelineAggregators, doHashCode()); } - // norelease: make this abstract when all InternalAggregations implement this method /** - * Opportunity for subclasses to the {@link #hashCode(Object)} for this + * Opportunity for subclasses to the {@link #hashCode()} for this * class. **/ protected int doHashCode() { @@ -223,7 +222,7 @@ public boolean equals(Object obj) { /** * Opportunity for subclasses to add criteria to the {@link #equals(Object)} * method for this class. - * + * * This method can safely cast obj to the subclass since the * {@link #equals(Object)} method checks that obj is the same * class as this From fd6e1a30de3cdc68d9ee58eefee40e5055b7487b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 22 Dec 2016 14:34:08 +0100 Subject: [PATCH 11/22] Improve concurrency of ShardCoreKeyMap. (#22316) `ShardCoreKeyMap.add` is called on each segment for all search requests, which means it might become a bottleneck under a cocurrent load of cheap search requests since this method acquires a mutex. This change proposes to use a `ConcurrentHashMap` which allows to only take the mutex in the case that the `LeafReader` has never been seen before. --- .../common/lucene/ShardCoreKeyMap.java | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java index 146fb7ba05ec4..38e7691f398f1 100644 --- a/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java +++ b/core/src/main/java/org/elasticsearch/common/lucene/ShardCoreKeyMap.java @@ -29,9 +29,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * A map between segment core cache keys and the shard that these segments @@ -50,7 +50,7 @@ public final class ShardCoreKeyMap { private final Map> indexToCoreKey; public ShardCoreKeyMap() { - coreKeyToShard = new IdentityHashMap<>(); + coreKeyToShard = new ConcurrentHashMap<>(); indexToCoreKey = new HashMap<>(); } @@ -64,9 +64,17 @@ public void add(LeafReader reader) { throw new IllegalArgumentException("Could not extract shard id from " + reader); } final Object coreKey = reader.getCoreCacheKey(); + + if (coreKeyToShard.containsKey(coreKey)) { + // Do this check before entering the synchronized block in order to + // avoid taking the mutex if possible (which should happen most of + // the time). + return; + } + final String index = shardId.getIndexName(); synchronized (this) { - if (coreKeyToShard.put(coreKey, shardId) == null) { + if (coreKeyToShard.containsKey(coreKey) == false) { Set objects = indexToCoreKey.get(index); if (objects == null) { objects = new HashSet<>(); @@ -90,6 +98,14 @@ public void add(LeafReader reader) { try { reader.addCoreClosedListener(listener); addedListener = true; + + // Only add the core key to the map as a last operation so that + // if another thread sees that the core key is already in the + // map (like the check just before this synchronized block), + // then it means that the closed listener has already been + // registered. + ShardId previous = coreKeyToShard.put(coreKey, shardId); + assert previous == null; } finally { if (false == addedListener) { try { From e39942fc02058367aa8d5d938986613687b66483 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 22 Dec 2016 14:35:12 +0100 Subject: [PATCH 12/22] `value_type` is useful regardless of scripting. (#22160) Today we only expose `value_type` in scriptable aggregations, however it is also useful with unmapped fields. I suspect we never noticed because `value_type` was not documented (fixed) and most aggregations are scriptable. Closes #20163 --- .../aggregations/support/MissingValues.java | 2 +- .../support/ValuesSourceParserHelper.java | 22 +-- .../support/MissingValuesTests.java | 5 + docs/reference/aggregations.asciidoc | 14 +- .../test/search.aggregation/20_terms.yaml | 129 ++++++++++++++++++ 5 files changed, 155 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 28a4bd2567ce4..4a01b67b78078 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -80,7 +80,7 @@ public int count() { } public static ValuesSource.Numeric replaceMissing(final ValuesSource.Numeric valuesSource, final Number missing) { - final boolean missingIsFloat = missing.longValue() != (long) missing.doubleValue(); + final boolean missingIsFloat = missing.doubleValue() % 1 != 0; final boolean isFloatingPoint = valuesSource.isFloatingPoint() || missingIsFloat; return new ValuesSource.Numeric() { diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java index 4116d30a658e3..2af21192a446b 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java @@ -67,6 +67,17 @@ private static void declareFields( objectParser.declareField(ValuesSourceAggregationBuilder::missing, XContentParser::objectText, new ParseField("missing"), ObjectParser.ValueType.VALUE); + objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> { + ValueType valueType = ValueType.resolveForScript(p.text()); + if (targetValueType != null && valueType.isNotA(targetValueType)) { + throw new ParsingException(p.getTokenLocation(), + "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type [" + + valueType + "]. It can only work on value of type [" + + targetValueType + "]"); + } + return valueType; + }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING); + if (formattable) { objectParser.declareField(ValuesSourceAggregationBuilder::format, XContentParser::text, new ParseField("format"), ObjectParser.ValueType.STRING); @@ -75,17 +86,6 @@ private static void declareFields( if (scriptable) { objectParser.declareField(ValuesSourceAggregationBuilder::script, org.elasticsearch.script.Script::parse, Script.SCRIPT_PARSE_FIELD, ObjectParser.ValueType.OBJECT_OR_STRING); - - objectParser.declareField(ValuesSourceAggregationBuilder::valueType, p -> { - ValueType valueType = ValueType.resolveForScript(p.text()); - if (targetValueType != null && valueType.isNotA(targetValueType)) { - throw new ParsingException(p.getTokenLocation(), - "Aggregation [" + objectParser.getName() + "] was configured with an incompatible value type [" - + valueType + "]. It can only work on value of type [" - + targetValueType + "]"); - } - return valueType; - }, new ParseField("value_type", "valueType"), ObjectParser.ValueType.STRING); } if (timezoneAware) { diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 3b4e84f8e7cca..2c819a671e55d 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -293,4 +293,9 @@ public int count() { } } } + + public void testFloatingPointDetection() { + assertFalse(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3).isFloatingPoint()); + assertTrue(MissingValues.replaceMissing(ValuesSource.Numeric.EMPTY, 3.5).isFloatingPoint()); + } } diff --git a/docs/reference/aggregations.asciidoc b/docs/reference/aggregations.asciidoc index 2b6523725b669..586e582a05f8f 100644 --- a/docs/reference/aggregations.asciidoc +++ b/docs/reference/aggregations.asciidoc @@ -92,11 +92,15 @@ enables defining all the "dynamic" expressions in the script as parameters, whic between calls (this will ensure the use of the cached compiled scripts in Elasticsearch). =============================== -Scripts can generate a single value or multiple values per document. When generating multiple values, one can use the -`script_values_sorted` settings to indicate whether these values are sorted or not. Internally, Elasticsearch can -perform optimizations when dealing with sorted values (for example, with the `min` aggregations, knowing the values are -sorted, Elasticsearch will skip the iterations over all the values and rely on the first value in the list to be the -minimum value among all other values associated with the same document). +Elasticsearch uses the type of the field in the mapping in order to figure out +how to run the aggregation and format the response. However there are two cases +in which Elasticsearch cannot figure out this information: unmapped fields (for +instance in the case of a search request across multiple indices, and only some +of them have a mapping for the field) and pure scripts. For those cases, it is +possible to give Elasticsearch a hint using the `value_type` option, which +accepts the following values: `string`, `long` (works for all integer types), +`double` (works for all decimal types like `float` or `scaled_float`), `date`, +`ip` and `boolean`. -- diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml index c52723a14133c..35febfa28da29 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/20_terms.yaml @@ -465,3 +465,132 @@ setup: - match: { aggregations.str_terms.buckets.0.key: 1234 } - match: { aggregations.str_terms.buckets.0.doc_count: 2 } + +--- +"Unmapped strings": + + - do: + index: + index: test_1 + type: test + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + search: + body: { "size" : 0, "aggs" : { "string_terms" : { "terms" : { "field" : "unmapped_string", "value_type" : "string", "missing": "abc" } } } } + + - match: { hits.total: 1 } + + - length: { aggregations.string_terms.buckets: 1 } + + - match: { aggregations.string_terms.buckets.0.key: "abc" } + + - match: { aggregations.string_terms.buckets.0.doc_count: 1 } + +--- +"Unmapped booleans": + + - do: + index: + index: test_1 + type: test + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + search: + body: { "size" : 0, "aggs" : { "boolean_terms" : { "terms" : { "field" : "unmapped_boolean", "value_type" : "boolean", "missing": true } } } } + + - match: { hits.total: 1 } + + - length: { aggregations.boolean_terms.buckets: 1 } + + - match: { aggregations.boolean_terms.buckets.0.key: 1 } + + - match: { aggregations.boolean_terms.buckets.0.key_as_string: "true" } + + - match: { aggregations.boolean_terms.buckets.0.doc_count: 1 } + +--- +"Unmapped dates": + + - do: + index: + index: test_1 + type: test + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + search: + body: { "size" : 0, "aggs" : { "date_terms" : { "terms" : { "field" : "unmapped_date", "value_type" : "date", "missing": "2016-05-11" } } } } + + - match: { hits.total: 1 } + + - length: { aggregations.date_terms.buckets: 1 } + + - match: { aggregations.date_terms.buckets.0.key: 1462924800000 } + + - match: { aggregations.date_terms.buckets.0.key_as_string: "2016-05-11T00:00:00.000Z" } + + - match: { aggregations.date_terms.buckets.0.doc_count: 1 } + +--- +"Unmapped longs": + + - do: + index: + index: test_1 + type: test + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + search: + body: { "size" : 0, "aggs" : { "long_terms" : { "terms" : { "field" : "unmapped_long", "value_type" : "long", "missing": 3 } } } } + + - match: { hits.total: 1 } + + - length: { aggregations.long_terms.buckets: 1 } + + - match: { aggregations.long_terms.buckets.0.key: 3 } + + - match: { aggregations.long_terms.buckets.0.doc_count: 1 } + +--- +"Unmapped doubles": + + - do: + index: + index: test_1 + type: test + id: 1 + body: {} + + - do: + indices.refresh: {} + + - do: + search: + body: { "size" : 0, "aggs" : { "double_terms" : { "terms" : { "field" : "unmapped_double", "value_type" : "double", "missing": 3.5 } } } } + + - match: { hits.total: 1 } + + - length: { aggregations.double_terms.buckets: 1 } + + - match: { aggregations.double_terms.buckets.0.key: 3.5 } + + - match: { aggregations.double_terms.buckets.0.doc_count: 1 } From 9b3b693d15a230cf0af6fc44e8dea0e31839c259 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 22 Dec 2016 14:35:59 +0100 Subject: [PATCH 13/22] Date detection should not rely on a hardcoded set of characters. (#22171) Currently we only apply date detection on strings that contain either `:`, `-` or `/`. This commit inverses the heuristic in order to only apply date detection on strings that are not parseable as a number, so that more date formats can be used as dynamic dates formats. Closes #1694 --- .../org/elasticsearch/common/Strings.java | 20 ----- .../index/mapper/DocumentParser.java | 77 ++++++++++--------- .../index/mapper/DocumentParserTests.java | 46 +++++++++++ 3 files changed, 88 insertions(+), 55 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/Strings.java b/core/src/main/java/org/elasticsearch/common/Strings.java index 1ef13e3bc7057..1689054e043c3 100644 --- a/core/src/main/java/org/elasticsearch/common/Strings.java +++ b/core/src/main/java/org/elasticsearch/common/Strings.java @@ -276,26 +276,6 @@ public static boolean substringMatch(CharSequence str, int index, CharSequence s return true; } - /** - * Count the occurrences of the substring in string s. - * - * @param str string to search in. Return 0 if this is null. - * @param sub string to search for. Return 0 if this is null. - */ - public static int countOccurrencesOf(String str, String sub) { - if (str == null || sub == null || str.length() == 0 || sub.length() == 0) { - return 0; - } - int count = 0; - int pos = 0; - int idx; - while ((idx = str.indexOf(sub, pos)) != -1) { - ++count; - pos = idx + sub.length(); - } - return count; - } - /** * Replace all occurrences of a substring within a string with * another string. diff --git a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index d88ef2c7f4434..d7f32f4663bb9 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.joda.FormatDateTimeFormatter; import org.elasticsearch.common.xcontent.XContentHelper; @@ -650,47 +649,55 @@ private static Mapper.Builder createBuilderFromFieldType(final ParseContext private static Mapper.Builder createBuilderFromDynamicValue(final ParseContext context, XContentParser.Token token, String currentFieldName) throws IOException { if (token == XContentParser.Token.VALUE_STRING) { - if (context.root().dateDetection()) { - String text = context.parser().text(); - // a safe check since "1" gets parsed as well - if (Strings.countOccurrencesOf(text, ":") > 1 || Strings.countOccurrencesOf(text, "-") > 1 || Strings.countOccurrencesOf(text, "/") > 1) { - for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { - try { - dateTimeFormatter.parser().parseMillis(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE); - if (builder == null) { - builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings())); - } - return builder; - } catch (Exception e) { - // failure to parse this, continue - } - } - } + String text = context.parser().text(); + + boolean parseableAsLong = false; + try { + Long.parseLong(text); + parseableAsLong = true; + } catch (NumberFormatException e) { + // not a long number } - if (context.root().numericDetection()) { - String text = context.parser().text(); - try { - Long.parseLong(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG); - if (builder == null) { - builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings())); - } - return builder; - } catch (NumberFormatException e) { - // not a long number + + boolean parseableAsDouble = false; + try { + Double.parseDouble(text); + parseableAsDouble = true; + } catch (NumberFormatException e) { + // not a double number + } + + if (parseableAsLong && context.root().numericDetection()) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG); + if (builder == null) { + builder = newLongBuilder(currentFieldName, Version.indexCreated(context.indexSettings())); } - try { - Double.parseDouble(text); - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE); + return builder; + } else if (parseableAsDouble && context.root().numericDetection()) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE); + if (builder == null) { + builder = newFloatBuilder(currentFieldName, Version.indexCreated(context.indexSettings())); + } + return builder; + } else if (parseableAsLong == false && parseableAsDouble == false && context.root().dateDetection()) { + // We refuse to match pure numbers, which are too likely to be + // false positives with date formats that include eg. + // `epoch_millis` or `YYYY` + for (FormatDateTimeFormatter dateTimeFormatter : context.root().dynamicDateTimeFormatters()) { + try { + dateTimeFormatter.parser().parseMillis(text); + } catch (IllegalArgumentException e) { + // failure to parse this, continue + continue; + } + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DATE); if (builder == null) { - builder = newFloatBuilder(currentFieldName, Version.indexCreated(context.indexSettings())); + builder = newDateBuilder(currentFieldName, dateTimeFormatter, Version.indexCreated(context.indexSettings())); } return builder; - } catch (NumberFormatException e) { - // not a long number } } + Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING); if (builder == null) { builder = new TextFieldMapper.Builder(currentFieldName) diff --git a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index f5a92d3f9797e..f964492ea6644 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -49,6 +49,7 @@ import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; // TODO: make this a real unit test public class DocumentParserTests extends ESSingleNodeTestCase { @@ -1260,4 +1261,49 @@ public void testIncludeInAllPropagation() throws IOException { } assertEquals(new HashSet<>(Arrays.asList("b", "d")), values); } + + public void testDynamicDateDetectionDisabledOnNumbers() throws IOException { + DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_date_formats") + .value("yyyy") + .endArray().endObject().endObject().string(); + DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping)); + + BytesReference bytes = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "2016") + .endObject().bytes(); + + // Even though we matched the dynamic format, we do not match on numbers, + // which are too likely to be false positives + ParsedDocument doc = mapper.parse("test", "type", "1", bytes); + Mapping update = doc.dynamicMappingsUpdate(); + assertNotNull(update); + Mapper dateMapper = update.root().getMapper("foo"); + assertNotNull(dateMapper); + assertThat(dateMapper, not(instanceOf(DateFieldMapper.class))); + } + + public void testDynamicDateDetectionEnabledWithNoSpecialCharacters() throws IOException { + DocumentMapperParser mapperParser = createIndex("test").mapperService().documentMapperParser(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startArray("dynamic_date_formats") + .value("yyyy MM") + .endArray().endObject().endObject().string(); + DocumentMapper mapper = mapperParser.parse("type", new CompressedXContent(mapping)); + + BytesReference bytes = XContentFactory.jsonBuilder() + .startObject() + .field("foo", "2016 12") + .endObject().bytes(); + + // We should have generated a date field + ParsedDocument doc = mapper.parse("test", "type", "1", bytes); + Mapping update = doc.dynamicMappingsUpdate(); + assertNotNull(update); + Mapper dateMapper = update.root().getMapper("foo"); + assertNotNull(dateMapper); + assertThat(dateMapper, instanceOf(DateFieldMapper.class)); + } } From 9a73a2efb302e008bbff1ba172c7f99e414dc146 Mon Sep 17 00:00:00 2001 From: Colin Goodheart-Smithe Date: Thu, 22 Dec 2016 13:50:54 +0000 Subject: [PATCH 14/22] Fix stackoverflow error on InternalNumericMetricAggregation --- .../metrics/InternalNumericMetricsAggregation.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java index b9442637a6bb9..010e24346bfad 100644 --- a/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java +++ b/core/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalNumericMetricsAggregation.java @@ -105,12 +105,18 @@ protected InternalNumericMetricsAggregation(StreamInput in) throws IOException { } @Override - protected int doHashCode() { + public int hashCode() { return Objects.hash(format, super.hashCode()); } @Override - protected boolean doEquals(Object obj) { + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } InternalNumericMetricsAggregation other = (InternalNumericMetricsAggregation) obj; return super.equals(obj) && Objects.equals(format, other.format); From b9a90eca20b2406ee7af8033ff6a2a445603520d Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 21 Dec 2016 17:24:32 +0100 Subject: [PATCH 15/22] inner hits: Don't inline inner hits if the query the inner hits is inlined into can't resolve mappings and ignore_unmapped has been set to true Closes #21620 --- .../index/query/HasChildQueryBuilder.java | 12 +- .../index/query/HasParentQueryBuilder.java | 8 +- .../index/query/InnerHitBuilder.java | 53 ++++++-- .../index/query/NestedQueryBuilder.java | 6 +- .../index/query/QueryShardContext.java | 9 ++ .../query/HasChildQueryBuilderTests.java | 7 +- .../query/HasParentQueryBuilderTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 83 +++++++++++-- .../index/query/NestedQueryBuilderTests.java | 4 +- .../search/child/ChildQuerySearchIT.java | 4 +- .../search/fetch/subphase/InnerHitsIT.java | 116 ++++++++++++------ 11 files changed, 228 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java index 85b4fd18da51d..eb2ae57f6dfe4 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasChildQueryBuilder.java @@ -145,8 +145,8 @@ public InnerHitBuilder innerHit() { return innerHitBuilder; } - public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit) { - this.innerHitBuilder = new InnerHitBuilder(Objects.requireNonNull(innerHit), query, type); + public HasChildQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) { + this.innerHitBuilder = new InnerHitBuilder(Objects.requireNonNull(innerHit), query, type, ignoreUnmapped); return this; } @@ -268,13 +268,13 @@ public static HasChildQueryBuilder fromXContent(QueryParseContext parseContext) } } HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder(childType, iqb, scoreMode); - if (innerHitBuilder != null) { - hasChildQueryBuilder.innerHit(innerHitBuilder); - } hasChildQueryBuilder.minMaxChildren(minChildren, maxChildren); hasChildQueryBuilder.queryName(queryName); hasChildQueryBuilder.boost(boost); hasChildQueryBuilder.ignoreUnmapped(ignoreUnmapped); + if (innerHitBuilder != null) { + hasChildQueryBuilder.innerHit(innerHitBuilder, ignoreUnmapped); + } return hasChildQueryBuilder; } @@ -318,7 +318,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { context.setTypes(previousTypes); } - DocumentMapper childDocMapper = context.getMapperService().documentMapper(type); + DocumentMapper childDocMapper = context.documentMapper(type); if (childDocMapper == null) { if (ignoreUnmapped) { return new MatchNoDocsQuery(); diff --git a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java index f8644a87528bb..8011f8798fbd9 100644 --- a/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/HasParentQueryBuilder.java @@ -124,8 +124,8 @@ public InnerHitBuilder innerHit() { return innerHit; } - public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit) { - this.innerHit = new InnerHitBuilder(innerHit, query, type); + public HasParentQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) { + this.innerHit = new InnerHitBuilder(innerHit, query, type, ignoreUnmapped); return this; } @@ -159,7 +159,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { context.setTypes(previousTypes); } - DocumentMapper parentDocMapper = context.getMapperService().documentMapper(type); + DocumentMapper parentDocMapper = context.documentMapper(type); if (parentDocMapper == null) { if (ignoreUnmapped) { return new MatchNoDocsQuery(); @@ -276,7 +276,7 @@ public static HasParentQueryBuilder fromXContent(QueryParseContext parseContext) .queryName(queryName) .boost(boost); if (innerHits != null) { - queryBuilder.innerHit(innerHits); + queryBuilder.innerHit(innerHits, ignoreUnmapped); } return queryBuilder; } diff --git a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 3a98ba36d204c..5bd7b20cb29a4 100644 --- a/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.query; +import org.elasticsearch.Version; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -45,7 +46,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -59,6 +59,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeable { public static final ParseField NAME_FIELD = new ParseField("name"); + public static final ParseField IGNORE_UNMAPPED = new ParseField("ignore_unmapped"); public static final ParseField INNER_HITS_FIELD = new ParseField("inner_hits"); public static final QueryBuilder DEFAULT_INNER_HIT_QUERY = new MatchAllQueryBuilder(); @@ -66,6 +67,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl static { PARSER.declareString(InnerHitBuilder::setName, NAME_FIELD); + PARSER.declareBoolean((innerHitBuilder, value) -> innerHitBuilder.ignoreUnmapped = value, IGNORE_UNMAPPED); PARSER.declareInt(InnerHitBuilder::setFrom, SearchSourceBuilder.FROM_FIELD); PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD); PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD); @@ -130,6 +132,7 @@ public final class InnerHitBuilder extends ToXContentToBytes implements Writeabl private String name; private String nestedPath; private String parentChildType; + private boolean ignoreUnmapped; private int from; private int size = 3; @@ -151,6 +154,7 @@ public InnerHitBuilder() { private InnerHitBuilder(InnerHitBuilder other) { name = other.name; + this.ignoreUnmapped = other.ignoreUnmapped; from = other.from; size = other.size; explain = other.explain; @@ -180,19 +184,21 @@ private InnerHitBuilder(InnerHitBuilder other) { } - InnerHitBuilder(InnerHitBuilder other, String nestedPath, QueryBuilder query) { + InnerHitBuilder(InnerHitBuilder other, String nestedPath, QueryBuilder query, boolean ignoreUnmapped) { this(other); this.query = query; this.nestedPath = nestedPath; + this.ignoreUnmapped = ignoreUnmapped; if (name == null) { this.name = nestedPath; } } - InnerHitBuilder(InnerHitBuilder other, QueryBuilder query, String parentChildType) { + InnerHitBuilder(InnerHitBuilder other, QueryBuilder query, String parentChildType, boolean ignoreUnmapped) { this(other); this.query = query; this.parentChildType = parentChildType; + this.ignoreUnmapped = ignoreUnmapped; if (name == null) { this.name = parentChildType; } @@ -205,6 +211,9 @@ public InnerHitBuilder(StreamInput in) throws IOException { name = in.readOptionalString(); nestedPath = in.readOptionalString(); parentChildType = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) { + ignoreUnmapped = in.readBoolean(); + } from = in.readVInt(); size = in.readVInt(); explain = in.readBoolean(); @@ -243,6 +252,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(name); out.writeOptionalString(nestedPath); out.writeOptionalString(parentChildType); + if (out.getVersion().onOrAfter(Version.V_5_2_0_UNRELEASED)) { + out.writeBoolean(ignoreUnmapped); + } out.writeVInt(from); out.writeVInt(size); out.writeBoolean(explain); @@ -289,6 +301,13 @@ public InnerHitBuilder setName(String name) { return this; } + /** + * Whether to include inner hits in the search response hits if required mappings is missing + */ + public boolean isIgnoreUnmapped() { + return ignoreUnmapped; + } + public int getFrom() { return from; } @@ -523,6 +542,14 @@ public InnerHitsContext.BaseInnerHits build(SearchContext parentSearchContext, QueryShardContext queryShardContext = parentSearchContext.getQueryShardContext(); if (nestedPath != null) { ObjectMapper nestedObjectMapper = queryShardContext.getObjectMapper(nestedPath); + if (nestedObjectMapper == null) { + if (ignoreUnmapped == false) { + throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + nestedPath + "]"); + } else { + return null; + } + } + ObjectMapper parentObjectMapper = queryShardContext.nestedScope().nextLevel(nestedObjectMapper); InnerHitsContext.NestedInnerHits nestedInnerHits = new InnerHitsContext.NestedInnerHits( name, parentSearchContext, parentObjectMapper, nestedObjectMapper @@ -535,7 +562,15 @@ public InnerHitsContext.BaseInnerHits build(SearchContext parentSearchContext, innerHitsContext.addInnerHitDefinition(nestedInnerHits); return nestedInnerHits; } else if (parentChildType != null) { - DocumentMapper documentMapper = queryShardContext.getMapperService().documentMapper(parentChildType); + DocumentMapper documentMapper = queryShardContext.documentMapper(parentChildType); + if (documentMapper == null) { + if (ignoreUnmapped == false) { + throw new IllegalStateException("[" + query.getName() + "] no mapping found for type [" + parentChildType + "]"); + } else { + return null; + } + } + InnerHitsContext.ParentChildInnerHits parentChildInnerHits = new InnerHitsContext.ParentChildInnerHits( name, parentSearchContext, queryShardContext.getMapperService(), documentMapper ); @@ -556,7 +591,9 @@ private void buildChildInnerHits(SearchContext parentSearchContext, InnerHitsCon InnerHitsContext.BaseInnerHits childInnerHit = entry.getValue().build( parentSearchContext, new InnerHitsContext() ); - childInnerHits.put(entry.getKey(), childInnerHit); + if (childInnerHit != null) { + childInnerHits.put(entry.getKey(), childInnerHit); + } } innerHits.setChildInnerHits(childInnerHits); } @@ -617,6 +654,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (name != null) { builder.field(NAME_FIELD.getPreferredName(), name); } + builder.field(IGNORE_UNMAPPED.getPreferredName(), ignoreUnmapped); builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); @@ -672,6 +710,7 @@ public boolean equals(Object o) { return Objects.equals(name, that.name) && Objects.equals(nestedPath, that.nestedPath) && Objects.equals(parentChildType, that.parentChildType) && + Objects.equals(ignoreUnmapped, that.ignoreUnmapped) && Objects.equals(from, that.from) && Objects.equals(size, that.size) && Objects.equals(explain, that.explain) && @@ -689,8 +728,8 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, nestedPath, parentChildType, from, size, explain, version, trackScores, storedFieldsContext, - docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits); + return Objects.hash(name, nestedPath, parentChildType, ignoreUnmapped, from, size, explain, version, trackScores, + storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, query, childInnerHits); } public static InnerHitBuilder fromXContent(QueryParseContext context) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index 6c1a8b27f6b79..519abe41107a2 100644 --- a/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/core/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -103,8 +103,8 @@ public InnerHitBuilder innerHit() { return innerHitBuilder; } - public NestedQueryBuilder innerHit(InnerHitBuilder innerHit) { - this.innerHitBuilder = new InnerHitBuilder(innerHit, path, query); + public NestedQueryBuilder innerHit(InnerHitBuilder innerHit, boolean ignoreUnmapped) { + this.innerHitBuilder = new InnerHitBuilder(innerHit, path, query, ignoreUnmapped); return this; } @@ -194,7 +194,7 @@ public static NestedQueryBuilder fromXContent(QueryParseContext parseContext) th .queryName(queryName) .boost(boost); if (innerHitBuilder != null) { - queryBuilder.innerHit(innerHitBuilder); + queryBuilder.innerHit(innerHitBuilder, ignoreUnmapped); } return queryBuilder; } diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 31668922e6e4f..e075368a2b421 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; @@ -207,6 +208,14 @@ public ObjectMapper getObjectMapper(String name) { return mapperService.getObjectMapper(name); } + /** + * Returns s {@link DocumentMapper} instance for the given type. + * Delegates to {@link MapperService#documentMapper(String)} + */ + public DocumentMapper documentMapper(String type) { + return mapperService.documentMapper(type); + } + /** * Gets the search analyzer for the given field, or the default if there is none present for the field * TODO: remove this by moving defaults into mappers themselves diff --git a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java index 0caf112bd2673..e2c8e8b292cc0 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasChildQueryBuilderTests.java @@ -104,13 +104,13 @@ protected HasChildQueryBuilder doCreateTestQueryBuilder() { HasChildQueryBuilder hqb = new HasChildQueryBuilder(CHILD_TYPE, innerQueryBuilder, RandomPicks.randomFrom(random(), ScoreMode.values())); hqb.minMaxChildren(min, max); + hqb.ignoreUnmapped(randomBoolean()); if (randomBoolean()) { hqb.innerHit(new InnerHitBuilder() .setName(randomAsciiOfLengthBetween(1, 10)) .setSize(randomIntBetween(0, 100)) - .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))); + .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)), hqb.ignoreUnmapped()); } - hqb.ignoreUnmapped(randomBoolean()); return hqb; } @@ -189,6 +189,7 @@ public void testFromJson() throws IOException { " \"_name\" : \"WNzYMJKRwePuRBh\",\n" + " \"inner_hits\" : {\n" + " \"name\" : \"inner_hits_name\",\n" + + " \"ignore_unmapped\" : false,\n" + " \"from\" : 0,\n" + " \"size\" : 100,\n" + " \"version\" : false,\n" + @@ -211,7 +212,7 @@ public void testFromJson() throws IOException { assertEquals(query, queryBuilder.childType(), "child"); assertEquals(query, queryBuilder.scoreMode(), ScoreMode.Avg); assertNotNull(query, queryBuilder.innerHit()); - InnerHitBuilder expected = new InnerHitBuilder(new InnerHitBuilder(), queryBuilder.query(), "child") + InnerHitBuilder expected = new InnerHitBuilder(new InnerHitBuilder(), queryBuilder.query(), "child", false) .setName("inner_hits_name") .setSize(100) .addSort(new FieldSortBuilder("mapped_string").order(SortOrder.ASC)); diff --git a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java index cf4b0617ea98e..ffcfe701dceeb 100644 --- a/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/HasParentQueryBuilderTests.java @@ -85,13 +85,13 @@ protected HasParentQueryBuilder doCreateTestQueryBuilder() { innerQueryBuilder = new WrapperQueryBuilder(innerQueryBuilder.toString()); } HasParentQueryBuilder hqb = new HasParentQueryBuilder(PARENT_TYPE, innerQueryBuilder, randomBoolean()); + hqb.ignoreUnmapped(randomBoolean()); if (randomBoolean()) { hqb.innerHit(new InnerHitBuilder() .setName(randomAsciiOfLengthBetween(1, 10)) .setSize(randomIntBetween(0, 100)) - .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC))); + .addSort(new FieldSortBuilder(STRING_FIELD_NAME_2).order(SortOrder.ASC)), hqb.ignoreUnmapped()); } - hqb.ignoreUnmapped(randomBoolean()); return hqb; } diff --git a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 6b9fbe76ba83a..1f602a0d14cd5 100644 --- a/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -34,7 +34,9 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -57,6 +59,8 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class InnerHitBuilderTests extends ESTestCase { @@ -120,7 +124,7 @@ public void testEqualsAndHashcode() throws IOException { public void testInlineLeafInnerHitsNestedQuery() throws Exception { InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); - nestedQueryBuilder.innerHit(leafInnerHits); + nestedQueryBuilder.innerHit(leafInnerHits, false); Map innerHitBuilders = new HashMap<>(); nestedQueryBuilder.extractInnerHitBuilders(innerHitBuilders); assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue()); @@ -129,7 +133,7 @@ public void testInlineLeafInnerHitsNestedQuery() throws Exception { public void testInlineLeafInnerHitsHasChildQuery() throws Exception { InnerHitBuilder leafInnerHits = randomInnerHits(); HasChildQueryBuilder hasChildQueryBuilder = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits); + .innerHit(leafInnerHits, false); Map innerHitBuilders = new HashMap<>(); hasChildQueryBuilder.extractInnerHitBuilders(innerHitBuilders); assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue()); @@ -138,7 +142,7 @@ public void testInlineLeafInnerHitsHasChildQuery() throws Exception { public void testInlineLeafInnerHitsHasParentQuery() throws Exception { InnerHitBuilder leafInnerHits = randomInnerHits(); HasParentQueryBuilder hasParentQueryBuilder = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false) - .innerHit(leafInnerHits); + .innerHit(leafInnerHits, false); Map innerHitBuilders = new HashMap<>(); hasParentQueryBuilder.extractInnerHitBuilders(innerHitBuilders); assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue()); @@ -147,7 +151,7 @@ public void testInlineLeafInnerHitsHasParentQuery() throws Exception { public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits); + .innerHit(leafInnerHits, false); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder); Map innerHitBuilders = new HashMap<>(); boolQueryBuilder.extractInnerHitBuilders(innerHitBuilders); @@ -157,7 +161,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits); + .innerHit(leafInnerHits, false); ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder); Map innerHitBuilders = new HashMap<>(); constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders); @@ -167,10 +171,10 @@ public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { InnerHitBuilder leafInnerHits1 = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits1); + .innerHit(leafInnerHits1, false); InnerHitBuilder leafInnerHits2 = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits2); + .innerHit(leafInnerHits2, false); BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2); Map innerHitBuilders = new HashMap<>(); constantScoreQueryBuilder.extractInnerHitBuilders(innerHitBuilders); @@ -181,13 +185,68 @@ public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() { InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) - .innerHit(leafInnerHits); + .innerHit(leafInnerHits, false); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder); Map innerHitBuilders = new HashMap<>(); ((AbstractQueryBuilder) functionScoreQueryBuilder).extractInnerHitBuilders(innerHitBuilders); assertThat(innerHitBuilders.get(leafInnerHits.getName()), notNullValue()); } + public void testBuild_ingoreUnmappedNestQuery() throws Exception { + QueryShardContext queryShardContext = mock(QueryShardContext.class); + when(queryShardContext.getObjectMapper("path")).thenReturn(null); + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); + + InnerHitBuilder leafInnerHits = randomInnerHits(); + NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); + query1.innerHit(leafInnerHits, false); + expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext())); + + NestedQueryBuilder query2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); + query2.innerHit(leafInnerHits, true); + InnerHitsContext innerHitsContext = new InnerHitsContext(); + query2.innerHit().build(searchContext, innerHitsContext); + assertThat(innerHitsContext.getInnerHits().size(), equalTo(0)); + } + + public void testBuild_ignoreUnmappedHasChildQuery() throws Exception { + QueryShardContext queryShardContext = mock(QueryShardContext.class); + when(queryShardContext.documentMapper("type")).thenReturn(null); + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); + + InnerHitBuilder leafInnerHits = randomInnerHits(); + HasChildQueryBuilder query1 = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None) + .innerHit(leafInnerHits, false); + expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext())); + + HasChildQueryBuilder query2 = new HasChildQueryBuilder("type", new MatchAllQueryBuilder(), ScoreMode.None) + .innerHit(leafInnerHits, true); + InnerHitsContext innerHitsContext = new InnerHitsContext(); + query2.innerHit().build(searchContext, innerHitsContext); + assertThat(innerHitsContext.getInnerHits().size(), equalTo(0)); + } + + public void testBuild_ingoreUnmappedHasParentQuery() throws Exception { + QueryShardContext queryShardContext = mock(QueryShardContext.class); + when(queryShardContext.documentMapper("type")).thenReturn(null); + SearchContext searchContext = mock(SearchContext.class); + when(searchContext.getQueryShardContext()).thenReturn(queryShardContext); + + InnerHitBuilder leafInnerHits = randomInnerHits(); + HasParentQueryBuilder query1 = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false) + .innerHit(leafInnerHits, false); + expectThrows(IllegalStateException.class, () -> query1.innerHit().build(searchContext, new InnerHitsContext())); + + HasParentQueryBuilder query2 = new HasParentQueryBuilder("type", new MatchAllQueryBuilder(), false) + .innerHit(leafInnerHits, true); + InnerHitsContext innerHitsContext = new InnerHitsContext(); + query2.innerHit().build(searchContext, innerHitsContext); + assertThat(innerHitsContext.getInnerHits().size(), equalTo(0)); + } + + public static InnerHitBuilder randomInnerHits() { return randomInnerHits(true, true); } @@ -236,9 +295,9 @@ public static InnerHitBuilder randomInnerHits(boolean recursive, boolean include if (includeQueryTypeOrPath) { QueryBuilder query = new MatchQueryBuilder(randomAsciiOfLengthBetween(1, 16), randomAsciiOfLengthBetween(1, 16)); if (randomBoolean()) { - return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query); + return new InnerHitBuilder(innerHits, randomAsciiOfLength(8), query, randomBoolean()); } else { - return new InnerHitBuilder(innerHits, query, randomAsciiOfLength(8)); + return new InnerHitBuilder(innerHits, query, randomAsciiOfLength(8), randomBoolean()); } } else { return innerHits; @@ -248,8 +307,8 @@ public static InnerHitBuilder randomInnerHits(boolean recursive, boolean include public void testCopyConstructor() throws Exception { InnerHitBuilder original = randomInnerHits(); InnerHitBuilder copy = original.getNestedPath() != null ? - new InnerHitBuilder(original, original.getNestedPath(), original.getQuery()) : - new InnerHitBuilder(original, original.getQuery(), original.getParentChildType()); + new InnerHitBuilder(original, original.getNestedPath(), original.getQuery(), original.isIgnoreUnmapped()) : + new InnerHitBuilder(original, original.getQuery(), original.getParentChildType(), original.isIgnoreUnmapped()); assertThat(copy, equalTo(original)); copy = mutate(copy); assertThat(copy, not(equalTo(original))); diff --git a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index 988f6f5c4ba28..c322e0a71907f 100644 --- a/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -73,13 +73,13 @@ protected NestedQueryBuilder doCreateTestQueryBuilder() { } NestedQueryBuilder nqb = new NestedQueryBuilder("nested1", innerQueryBuilder, RandomPicks.randomFrom(random(), ScoreMode.values())); + nqb.ignoreUnmapped(randomBoolean()); if (randomBoolean()) { nqb.innerHit(new InnerHitBuilder() .setName(randomAsciiOfLengthBetween(1, 10)) .setSize(randomIntBetween(0, 100)) - .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC))); + .addSort(new FieldSortBuilder(INT_FIELD_NAME).order(SortOrder.ASC)), nqb.ignoreUnmapped()); } - nqb.ignoreUnmapped(randomBoolean()); return nqb; } diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index c3a33e6740118..391d8be412e4f 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -94,7 +94,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -822,7 +821,8 @@ public void testHasChildInnerHitsHighlighting() throws Exception { SearchResponse searchResponse = client().prepareSearch("test").setQuery( hasChildQuery("child", matchQuery("c_field", "foo"), ScoreMode.None) .innerHit(new InnerHitBuilder().setHighlightBuilder( - new HighlightBuilder().field(new Field("c_field").highlightQuery(QueryBuilders.matchQuery("c_field", "bar")))))) + new HighlightBuilder().field(new Field("c_field") + .highlightQuery(QueryBuilders.matchQuery("c_field", "bar")))), false)) .get(); assertNoFailures(searchResponse); assertThat(searchResponse.getHits().totalHits(), equalTo(1L)); diff --git a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java index 809591cd6881b..c00882522243a 100644 --- a/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java +++ b/core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java @@ -123,7 +123,7 @@ public void testSimpleNested() throws Exception { SearchResponse response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setName("comment")) + .innerHit(new InnerHitBuilder().setName("comment"), false) ).get(); assertNoFailures(response); assertHitCount(response, 1); @@ -141,7 +141,7 @@ public void testSimpleNested() throws Exception { response = client().prepareSearch("articles") .setQuery(nestedQuery("comments", matchQuery("comments.message", "elephant"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setName("comment")) + .innerHit(new InnerHitBuilder().setName("comment"), false) ).get(); assertNoFailures(response); assertHitCount(response, 1); @@ -168,8 +168,8 @@ public void testSimpleNested() throws Exception { .addDocValueField("comments.message") .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())) - .setSize(1) - )).get(); + .setSize(1), + false)).get(); assertNoFailures(response); innerHits = response.getHits().getAt(0).getInnerHits().get("comments"); assertThat(innerHits.getTotalHits(), equalTo(2L)); @@ -207,9 +207,9 @@ public void testRandomNested() throws Exception { int size = randomIntBetween(0, numDocs); BoolQueryBuilder boolQuery = new BoolQueryBuilder(); boolQuery.should(nestedQuery("field1", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("a").setSize(size) - .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)))); + .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)), false)); boolQuery.should(nestedQuery("field2", matchAllQuery(), ScoreMode.Avg).innerHit(new InnerHitBuilder().setName("b") - .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size))); + .addSort(new FieldSortBuilder("_doc").order(SortOrder.DESC)).setSize(size), false)); SearchResponse searchResponse = client().prepareSearch("idx") .setQuery(boolQuery) .setSize(numDocs) @@ -260,7 +260,8 @@ public void testSimpleParentChild() throws Exception { indexRandom(true, requests); SearchResponse response = client().prepareSearch("articles") - .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(new InnerHitBuilder())) + .setQuery(hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None) + .innerHit(new InnerHitBuilder(), false)) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -277,7 +278,8 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(1).type(), equalTo("comment")); response = client().prepareSearch("articles") - .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None).innerHit(new InnerHitBuilder())) + .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None) + .innerHit(new InnerHitBuilder(), false)) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -302,8 +304,8 @@ public void testSimpleParentChild() throws Exception { .setHighlightBuilder(new HighlightBuilder().field("message")) .setExplain(true).setSize(1) .addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", - Collections.emptyMap())) - ) + Collections.emptyMap())), + false) ).get(); assertNoFailures(response); innerHits = response.getHits().getAt(0).getInnerHits().get("comment"); @@ -349,9 +351,11 @@ public void testRandomParentChild() throws Exception { int size = randomIntBetween(0, numDocs); BoolQueryBuilder boolQuery = new BoolQueryBuilder(); boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None) - .innerHit(new InnerHitBuilder().setName("a").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size)))); + .innerHit(new InnerHitBuilder().setName("a") + .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size), false))); boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None) - .innerHit(new InnerHitBuilder().setName("b").addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size)))); + .innerHit(new InnerHitBuilder().setName("b") + .addSort(new FieldSortBuilder("_uid").order(SortOrder.ASC)).setSize(size), false))); SearchResponse searchResponse = client().prepareSearch("idx") .setSize(numDocs) .setTypes("parent") @@ -417,7 +421,7 @@ public void testInnerHitsOnHasParent() throws Exception { .setQuery( boolQuery() .must(matchQuery("body", "fail2ban")) - .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder())) + .must(hasParentQuery("question", matchAllQuery(), false).innerHit(new InnerHitBuilder(), false)) ).get(); assertNoFailures(response); assertHitCount(response, 2); @@ -455,8 +459,8 @@ public void testParentChildMultipleLayers() throws Exception { SearchResponse response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", - hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder()), - ScoreMode.None).innerHit(new InnerHitBuilder())) + hasChildQuery("remark", matchQuery("message", "good"), ScoreMode.None).innerHit(new InnerHitBuilder(), false), + ScoreMode.None).innerHit(new InnerHitBuilder(), false)) .get(); assertNoFailures(response); @@ -476,8 +480,8 @@ public void testParentChildMultipleLayers() throws Exception { response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", - hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder()), - ScoreMode.None).innerHit(new InnerHitBuilder())) + hasChildQuery("remark", matchQuery("message", "bad"), ScoreMode.None).innerHit(new InnerHitBuilder(), false), + ScoreMode.None).innerHit(new InnerHitBuilder(), false)) .get(); assertNoFailures(response); @@ -543,8 +547,8 @@ public void testNestedMultipleLayers() throws Exception { .setQuery( nestedQuery("comments", nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "good"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setName("remark")), - ScoreMode.Avg).innerHit(new InnerHitBuilder()) + .innerHit(new InnerHitBuilder().setName("remark"), false), + ScoreMode.Avg).innerHit(new InnerHitBuilder(), false) ).get(); assertNoFailures(response); assertHitCount(response, 1); @@ -568,7 +572,7 @@ public void testNestedMultipleLayers() throws Exception { // Directly refer to the second level: response = client().prepareSearch("articles") .setQuery(nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder())).get(); + .innerHit(new InnerHitBuilder(), false)).get(); assertNoFailures(response); assertHitCount(response, 1); assertSearchHit(response, 1, hasId("2")); @@ -586,8 +590,8 @@ public void testNestedMultipleLayers() throws Exception { .setQuery( nestedQuery("comments", nestedQuery("comments.remarks", matchQuery("comments.remarks.message", "bad"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setName("remark")), - ScoreMode.Avg).innerHit(new InnerHitBuilder()) + .innerHit(new InnerHitBuilder().setName("remark"), false), + ScoreMode.Avg).innerHit(new InnerHitBuilder(), false) ).get(); assertNoFailures(response); assertHitCount(response, 1); @@ -621,7 +625,8 @@ public void testNestedDefinedAsObject() throws Exception { indexRandom(true, requests); SearchResponse response = client().prepareSearch("articles") - .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(new InnerHitBuilder())) + .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg) + .innerHit(new InnerHitBuilder(), false)) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -663,7 +668,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { SearchResponse response = client().prepareSearch("articles") .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder())).get(); + .innerHit(new InnerHitBuilder(), false)).get(); assertNoFailures(response); assertHitCount(response, 1); SearchHit hit = response.getHits().getAt(0); @@ -677,7 +682,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { response = client().prepareSearch("articles") .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "bear"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder())).get(); + .innerHit(new InnerHitBuilder(), false)).get(); assertNoFailures(response); assertHitCount(response, 1); hit = response.getHits().getAt(0); @@ -698,7 +703,7 @@ public void testInnerHitsWithObjectFieldThatHasANestedField() throws Exception { indexRandom(true, requests); response = client().prepareSearch("articles") .setQuery(nestedQuery("comments.messages", matchQuery("comments.messages.message", "fox"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder())).get(); + .innerHit(new InnerHitBuilder(), false)).get(); assertNoFailures(response); assertHitCount(response, 1); hit = response.getHits().getAt(0);; @@ -739,15 +744,16 @@ public void testRoyals() throws Exception { .setTypes("duke") .setQuery(boolQuery() .filter(hasParentQuery("prince", - hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings")), - false).innerHit(new InnerHitBuilder().setName("princes")) + hasParentQuery("king", matchAllQuery(), false).innerHit(new InnerHitBuilder().setName("kings"), false), + false).innerHit(new InnerHitBuilder().setName("princes"), false) ) .filter(hasChildQuery("earl", - hasChildQuery("baron", matchAllQuery(), ScoreMode.None).innerHit(new InnerHitBuilder().setName("barons")), + hasChildQuery("baron", matchAllQuery(), ScoreMode.None) + .innerHit(new InnerHitBuilder().setName("barons"), false), ScoreMode.None).innerHit(new InnerHitBuilder() .addSort(SortBuilders.fieldSort("_uid").order(SortOrder.ASC)) .setName("earls") - .setSize(4)) + .setSize(4), false) ) ) .get(); @@ -860,7 +866,7 @@ public void testMatchesQueriesNestedInnerHits() throws Exception { .should(termQuery("nested1.n_field1", "n_value1_3").queryName("test2")) .should(termQuery("nested1.n_field2", "n_value2_2").queryName("test3")); query = nestedQuery("nested1", query, ScoreMode.Avg).innerHit( - new InnerHitBuilder().addSort(new FieldSortBuilder("nested1.n_field1").order(SortOrder.ASC))); + new InnerHitBuilder().addSort(new FieldSortBuilder("nested1.n_field1").order(SortOrder.ASC)), false); SearchResponse searchResponse = client().prepareSearch("test") .setQuery(query) .setSize(numDocs) @@ -902,7 +908,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { SearchResponse response = client().prepareSearch("index") .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None) - .innerHit(new InnerHitBuilder())) + .innerHit(new InnerHitBuilder(), false)) .addSort("_uid", SortOrder.ASC) .get(); assertHitCount(response, 2); @@ -917,7 +923,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { assertThat(response.getHits().getAt(1).getInnerHits().get("child").getAt(0).getMatchedQueries()[0], equalTo("_name1")); QueryBuilder query = hasChildQuery("child", matchQuery("field", "value2").queryName("_name2"), ScoreMode.None) - .innerHit(new InnerHitBuilder()); + .innerHit(new InnerHitBuilder(), false); response = client().prepareSearch("index") .setQuery(query) .addSort("_uid", SortOrder.ASC) @@ -937,7 +943,7 @@ public void testDontExplode() throws Exception { indexRandom(true, requests); QueryBuilder query = hasChildQuery("child", matchQuery("field", "value1"), ScoreMode.None) - .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1)); + .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false); SearchResponse response = client().prepareSearch("index1") .setQuery(query) .addSort("_uid", SortOrder.ASC) @@ -957,7 +963,7 @@ public void testDontExplode() throws Exception { .get(); query = nestedQuery("nested", matchQuery("nested.field", "value1"), ScoreMode.Avg) - .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1)); + .innerHit(new InnerHitBuilder().setSize(ArrayUtil.MAX_ARRAY_LENGTH - 1), false); response = client().prepareSearch("index2") .setQuery(query) .addSort("_uid", SortOrder.ASC) @@ -983,7 +989,7 @@ public void testNestedSourceFiltering() throws Exception { SearchResponse response = client().prepareSearch() .setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.None) .innerHit(new InnerHitBuilder().setFetchSourceContext(new FetchSourceContext(true, - new String[]{"comments.message"}, null)))) + new String[]{"comments.message"}, null)), false)) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -1004,7 +1010,7 @@ public void testNestedInnerHitWrappedInParentChildInnerhit() throws Exception { SearchResponse response = client().prepareSearch("test") .setQuery(boolQuery().must(matchQuery("key", "value")) .should(hasChildQuery("child_type", nestedQuery("nested_type", matchAllQuery(), ScoreMode.None) - .innerHit(new InnerHitBuilder()), ScoreMode.None).innerHit(new InnerHitBuilder()))) + .innerHit(new InnerHitBuilder(), false), ScoreMode.None).innerHit(new InnerHitBuilder(), false))) .get(); assertHitCount(response, 1); SearchHit hit = response.getHits().getAt(0); @@ -1012,4 +1018,40 @@ public void testNestedInnerHitWrappedInParentChildInnerhit() throws Exception { assertThat(hit.getInnerHits().get("child_type").getAt(0).getInnerHits().get("nested_type").getAt(0).field("_parent"), nullValue()); } + public void testInnerHitsWithIgnoreUnmapped() throws Exception { + assertAcked(prepareCreate("index1") + .addMapping("parent_type", "nested_type", "type=nested") + .addMapping("child_type", "_parent", "type=parent_type") + ); + createIndex("index2"); + client().prepareIndex("index1", "parent_type", "1").setSource("nested_type", Collections.singletonMap("key", "value")).get(); + client().prepareIndex("index1", "child_type", "2").setParent("1").setSource("{}").get(); + client().prepareIndex("index2", "type", "3").setSource("key", "value").get(); + refresh(); + + SearchResponse response = client().prepareSearch("index1", "index2") + .setQuery(boolQuery() + .should(nestedQuery("nested_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) + .innerHit(new InnerHitBuilder(), true)) + .should(termQuery("key", "value")) + ) + .addSort("_uid", SortOrder.ASC) + .get(); + assertNoFailures(response); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("1")); + + response = client().prepareSearch("index1", "index2") + .setQuery(boolQuery() + .should(hasChildQuery("child_type", matchAllQuery(), ScoreMode.None).ignoreUnmapped(true) + .innerHit(new InnerHitBuilder(), true)) + .should(termQuery("key", "value")) + ) + .addSort("_uid", SortOrder.ASC) + .get(); + assertNoFailures(response); + assertHitCount(response, 2); + assertThat(response.getHits().getAt(0).getId(), equalTo("1")); + } + } From e1b8528ab8d5386eacad1514d0e679fa870a5fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Campinas?= Date: Thu, 22 Dec 2016 14:20:15 +0000 Subject: [PATCH 16/22] Support numeric bounds with decimal parts for long/integer/short/byte datatypes (#21972) Close #21600 --- .../index/mapper/NumberFieldMapper.java | 166 +++++++++++----- .../index/mapper/RangeFieldMapper.java | 4 +- .../index/mapper/ScaledFloatFieldMapper.java | 14 +- .../index/mapper/NumberFieldTypeTests.java | 177 +++++++++++++++--- 4 files changed, 281 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 33a6d481ae428..ac96fe8199ef4 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -39,6 +39,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.action.fieldstats.FieldStats; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; @@ -54,6 +55,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -145,7 +147,7 @@ public Mapper.Builder parse(String name, Map node, if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); } - builder.nullValue(type.parse(propNode)); + builder.nullValue(type.parse(propNode, false)); iterator.remove(); } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext)); @@ -162,8 +164,8 @@ public Mapper.Builder parse(String name, Map node, public enum NumberType { HALF_FLOAT("half_float", NumericType.HALF_FLOAT) { @Override - Float parse(Object value) { - return (Float) FLOAT.parse(value); + Float parse(Object value, boolean coerce) { + return (Float) FLOAT.parse(value, false); } @Override @@ -173,7 +175,7 @@ Float parse(XContentParser parser, boolean coerce) throws IOException { @Override Query termQuery(String field, Object value) { - float v = parse(value); + float v = parse(value, false); return HalfFloatPoint.newExactQuery(field, v); } @@ -181,7 +183,7 @@ Query termQuery(String field, Object value) { Query termsQuery(String field, List values) { float[] v = new float[values.size()]; for (int i = 0; i < values.size(); ++i) { - v[i] = parse(values.get(i)); + v[i] = parse(values.get(i), false); } return HalfFloatPoint.newSetQuery(field, v); } @@ -216,14 +218,14 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { - l = parse(lowerTerm); + l = parse(lowerTerm, false); if (includeLower) { l = nextDown(l); } l = HalfFloatPoint.nextUp(l); } if (upperTerm != null) { - u = parse(upperTerm); + u = parse(upperTerm, false); if (includeUpper) { u = nextUp(u); } @@ -270,7 +272,7 @@ FieldStats.Double stats(IndexReader reader, String fieldName, }, FLOAT("float", NumericType.FLOAT) { @Override - Float parse(Object value) { + Float parse(Object value, boolean coerce) { if (value instanceof Number) { return ((Number) value).floatValue(); } @@ -287,7 +289,7 @@ Float parse(XContentParser parser, boolean coerce) throws IOException { @Override Query termQuery(String field, Object value) { - float v = parse(value); + float v = parse(value, false); return FloatPoint.newExactQuery(field, v); } @@ -295,7 +297,7 @@ Query termQuery(String field, Object value) { Query termsQuery(String field, List values) { float[] v = new float[values.size()]; for (int i = 0; i < values.size(); ++i) { - v[i] = parse(values.get(i)); + v[i] = parse(values.get(i), false); } return FloatPoint.newSetQuery(field, v); } @@ -330,13 +332,13 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, float l = Float.NEGATIVE_INFINITY; float u = Float.POSITIVE_INFINITY; if (lowerTerm != null) { - l = parse(lowerTerm); + l = parse(lowerTerm, false); if (includeLower == false) { l = nextUp(l); } } if (upperTerm != null) { - u = parse(upperTerm); + u = parse(upperTerm, false); if (includeUpper == false) { u = nextDown(u); } @@ -382,7 +384,7 @@ FieldStats.Double stats(IndexReader reader, String fieldName, }, DOUBLE("double", NumericType.DOUBLE) { @Override - Double parse(Object value) { + Double parse(Object value, boolean coerce) { if (value instanceof Number) { return ((Number) value).doubleValue(); } @@ -399,7 +401,7 @@ Double parse(XContentParser parser, boolean coerce) throws IOException { @Override Query termQuery(String field, Object value) { - double v = parse(value); + double v = parse(value, false); return DoublePoint.newExactQuery(field, v); } @@ -407,7 +409,7 @@ Query termQuery(String field, Object value) { Query termsQuery(String field, List values) { double[] v = new double[values.size()]; for (int i = 0; i < values.size(); ++i) { - v[i] = parse(values.get(i)); + v[i] = parse(values.get(i), false); } return DoublePoint.newSetQuery(field, v); } @@ -442,13 +444,13 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, double l = Double.NEGATIVE_INFINITY; double u = Double.POSITIVE_INFINITY; if (lowerTerm != null) { - l = parse(lowerTerm); + l = parse(lowerTerm, false); if (includeLower == false) { l = nextUp(l); } } if (upperTerm != null) { - u = parse(upperTerm); + u = parse(upperTerm, false); if (includeUpper == false) { u = nextDown(u); } @@ -494,13 +496,13 @@ FieldStats.Double stats(IndexReader reader, String fieldName, }, BYTE("byte", NumericType.BYTE) { @Override - Byte parse(Object value) { + Byte parse(Object value, boolean coerce) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); if (doubleValue < Byte.MIN_VALUE || doubleValue > Byte.MAX_VALUE) { throw new IllegalArgumentException("Value [" + value + "] is out of range for a byte"); } - if (doubleValue % 1 != 0) { + if (!coerce && doubleValue % 1 != 0) { throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); } return ((Number) value).byteValue(); @@ -555,13 +557,13 @@ Number valueForSearch(Number value) { }, SHORT("short", NumericType.SHORT) { @Override - Short parse(Object value) { + Short parse(Object value, boolean coerce) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); if (doubleValue < Short.MIN_VALUE || doubleValue > Short.MAX_VALUE) { throw new IllegalArgumentException("Value [" + value + "] is out of range for a short"); } - if (doubleValue % 1 != 0) { + if (!coerce && doubleValue % 1 != 0) { throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); } return ((Number) value).shortValue(); @@ -616,13 +618,13 @@ Number valueForSearch(Number value) { }, INTEGER("integer", NumericType.INT) { @Override - Integer parse(Object value) { + Integer parse(Object value, boolean coerce) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); if (doubleValue < Integer.MIN_VALUE || doubleValue > Integer.MAX_VALUE) { throw new IllegalArgumentException("Value [" + value + "] is out of range for an integer"); } - if (doubleValue % 1 != 0) { + if (!coerce && doubleValue % 1 != 0) { throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); } return ((Number) value).intValue(); @@ -640,15 +642,30 @@ Integer parse(XContentParser parser, boolean coerce) throws IOException { @Override Query termQuery(String field, Object value) { - int v = parse(value); + if (hasDecimalPart(value)) { + return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); + } + int v = parse(value, true); return IntPoint.newExactQuery(field, v); } @Override Query termsQuery(String field, List values) { int[] v = new int[values.size()]; - for (int i = 0; i < values.size(); ++i) { - v[i] = parse(values.get(i)); + int upTo = 0; + + for (int i = 0; i < values.size(); i++) { + Object value = values.get(i); + if (!hasDecimalPart(value)) { + v[upTo++] = parse(value, true); + } + } + + if (upTo == 0) { + return Queries.newMatchNoDocsQuery("All values have a decimal part"); + } + if (upTo != v.length) { + v = Arrays.copyOf(v, upTo); } return IntPoint.newSetQuery(field, v); } @@ -659,8 +676,15 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, int l = Integer.MIN_VALUE; int u = Integer.MAX_VALUE; if (lowerTerm != null) { - l = parse(lowerTerm); - if (includeLower == false) { + l = parse(lowerTerm, true); + // if the lower bound is decimal: + // - if the bound is positive then we increment it: + // if lowerTerm=1.5 then the (inclusive) bound becomes 2 + // - if the bound is negative then we leave it as is: + // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue + boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); + if ((lowerTermHasDecimalPart == false && includeLower == false) || + (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { if (l == Integer.MAX_VALUE) { return new MatchNoDocsQuery(); } @@ -668,8 +692,10 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, } } if (upperTerm != null) { - u = parse(upperTerm); - if (includeUpper == false) { + u = parse(upperTerm, true); + boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); + if ((upperTermHasDecimalPart == false && includeUpper == false) || + (upperTermHasDecimalPart && signum(upperTerm) < 0)) { if (u == Integer.MIN_VALUE) { return new MatchNoDocsQuery(); } @@ -716,13 +742,13 @@ FieldStats.Long stats(IndexReader reader, String fieldName, }, LONG("long", NumericType.LONG) { @Override - Long parse(Object value) { + Long parse(Object value, boolean coerce) { if (value instanceof Number) { double doubleValue = ((Number) value).doubleValue(); if (doubleValue < Long.MIN_VALUE || doubleValue > Long.MAX_VALUE) { throw new IllegalArgumentException("Value [" + value + "] is out of range for a long"); } - if (doubleValue % 1 != 0) { + if (!coerce && doubleValue % 1 != 0) { throw new IllegalArgumentException("Value [" + value + "] has a decimal part"); } return ((Number) value).longValue(); @@ -740,15 +766,30 @@ Long parse(XContentParser parser, boolean coerce) throws IOException { @Override Query termQuery(String field, Object value) { - long v = parse(value); + if (hasDecimalPart(value)) { + return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part"); + } + long v = parse(value, true); return LongPoint.newExactQuery(field, v); } @Override Query termsQuery(String field, List values) { long[] v = new long[values.size()]; - for (int i = 0; i < values.size(); ++i) { - v[i] = parse(values.get(i)); + int upTo = 0; + + for (int i = 0; i < values.size(); i++) { + Object value = values.get(i); + if (!hasDecimalPart(value)) { + v[upTo++] = parse(value, true); + } + } + + if (upTo == 0) { + return Queries.newMatchNoDocsQuery("All values have a decimal part"); + } + if (upTo != v.length) { + v = Arrays.copyOf(v, upTo); } return LongPoint.newSetQuery(field, v); } @@ -759,8 +800,15 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, long l = Long.MIN_VALUE; long u = Long.MAX_VALUE; if (lowerTerm != null) { - l = parse(lowerTerm); - if (includeLower == false) { + l = parse(lowerTerm, true); + // if the lower bound is decimal: + // - if the bound is positive then we increment it: + // if lowerTerm=1.5 then the (inclusive) bound becomes 2 + // - if the bound is negative then we leave it as is: + // if lowerTerm=-1.5 then the (inclusive) bound becomes -1 due to the call to longValue + boolean lowerTermHasDecimalPart = hasDecimalPart(lowerTerm); + if ((lowerTermHasDecimalPart == false && includeLower == false) || + (lowerTermHasDecimalPart && signum(lowerTerm) > 0)) { if (l == Long.MAX_VALUE) { return new MatchNoDocsQuery(); } @@ -768,8 +816,10 @@ Query rangeQuery(String field, Object lowerTerm, Object upperTerm, } } if (upperTerm != null) { - u = parse(upperTerm); - if (includeUpper == false) { + u = parse(upperTerm, true); + boolean upperTermHasDecimalPart = hasDecimalPart(upperTerm); + if ((upperTermHasDecimalPart == false && includeUpper == false) || + (upperTermHasDecimalPart && signum(upperTerm) < 0)) { if (u == Long.MIN_VALUE) { return new MatchNoDocsQuery(); } @@ -827,7 +877,7 @@ FieldStats.Long stats(IndexReader reader, String fieldName, public final String typeName() { return name; } - /** Get the associated numerit type */ + /** Get the associated numeric type */ final NumericType numericType() { return numericType; } @@ -836,7 +886,7 @@ final NumericType numericType() { abstract Query rangeQuery(String field, Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper); abstract Number parse(XContentParser parser, boolean coerce) throws IOException; - abstract Number parse(Object value); + abstract Number parse(Object value, boolean coerce); public abstract List createFields(String name, Number value, boolean indexed, boolean docValued, boolean stored); abstract FieldStats stats(IndexReader reader, String fieldName, @@ -844,6 +894,38 @@ abstract FieldStats stats(IndexReader reader, String fieldName Number valueForSearch(Number value) { return value; } + + /** + * Returns true if the object is a number and has a decimal part + */ + boolean hasDecimalPart(Object number) { + if (number instanceof Number) { + double doubleValue = ((Number) number).doubleValue(); + return doubleValue % 1 != 0; + } + if (number instanceof BytesRef) { + number = ((BytesRef) number).utf8ToString(); + } + if (number instanceof String) { + return Double.parseDouble((String) number) % 1 != 0; + } + return false; + } + + /** + * Returns -1, 0, or 1 if the value is lower than, equal to, or greater than 0 + */ + double signum(Object value) { + if (value instanceof Number) { + double doubleValue = ((Number) value).doubleValue(); + return Math.signum(doubleValue); + } + if (value instanceof BytesRef) { + value = ((BytesRef) value).utf8ToString(); + } + return Math.signum(Double.parseDouble(value.toString())); + } + } public static final class NumberFieldType extends MappedFieldType { @@ -1014,7 +1096,7 @@ protected void parseCreateField(ParseContext context, List field } if (numericValue == null) { - numericValue = fieldType().type.parse(value); + numericValue = fieldType().type.parse(value, coerce.value()); } if (includeInAll) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 214cc52c3ee99..b0a809c12dfce 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -728,8 +728,8 @@ public Number parseTo(RangeFieldType fieldType, XContentParser parser, boolean c public Query rangeQuery(String field, Object from, Object to, boolean includeFrom, boolean includeTo, ShapeRelation relation, @Nullable DateTimeZone timeZone, @Nullable DateMathParser dateMathParser, QueryShardContext context) { - Number lower = from == null ? minValue() : numberType.parse(from); - Number upper = to == null ? maxValue() : numberType.parse(to); + Number lower = from == null ? minValue() : numberType.parse(from, false); + Number upper = to == null ? maxValue() : numberType.parse(to, false); if (relation == ShapeRelation.WITHIN) { return withinQuery(field, lower, upper, includeFrom, includeTo); } else if (relation == ShapeRelation.CONTAINS) { diff --git a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 2a76aa1addd60..c1ac00326a766 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -144,7 +144,7 @@ public Mapper.Builder parse(String name, Map node, if (propNode == null) { throw new MapperParsingException("Property [null_value] cannot be null."); } - builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode)); + builder.nullValue(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false)); iterator.remove(); } else if (propName.equals("ignore_malformed")) { builder.ignoreMalformed(TypeParsers.nodeBooleanValue("ignore_malformed", propNode, parserContext)); @@ -153,7 +153,7 @@ public Mapper.Builder parse(String name, Map node, builder.coerce(TypeParsers.nodeBooleanValue("coerce", propNode, parserContext)); iterator.remove(); } else if (propName.equals("scaling_factor")) { - builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode).doubleValue()); + builder.scalingFactor(NumberFieldMapper.NumberType.DOUBLE.parse(propNode, false).doubleValue()); iterator.remove(); } } @@ -207,7 +207,7 @@ public void checkCompatibility(MappedFieldType other, List conflicts, bo @Override public Query termQuery(Object value, QueryShardContext context) { failIfNotIndexed(); - double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue(); + double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue(); long scaledValue = Math.round(queryValue * scalingFactor); Query query = NumberFieldMapper.NumberType.LONG.termQuery(name(), scaledValue); if (boost() != 1f) { @@ -221,7 +221,7 @@ public Query termsQuery(List values, QueryShardContext context) { failIfNotIndexed(); List scaledValues = new ArrayList<>(values.size()); for (Object value : values) { - double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value).doubleValue(); + double queryValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false).doubleValue(); long scaledValue = Math.round(queryValue * scalingFactor); scaledValues.add(scaledValue); } @@ -237,7 +237,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower failIfNotIndexed(); Long lo = null; if (lowerTerm != null) { - double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm).doubleValue(); + double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(lowerTerm, false).doubleValue(); if (includeLower == false) { dValue = Math.nextUp(dValue); } @@ -245,7 +245,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } Long hi = null; if (upperTerm != null) { - double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm).doubleValue(); + double dValue = NumberFieldMapper.NumberType.DOUBLE.parse(upperTerm, false).doubleValue(); if (includeUpper == false) { dValue = Math.nextDown(dValue); } @@ -404,7 +404,7 @@ protected void parseCreateField(ParseContext context, List field } if (numericValue == null) { - numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value); + numericValue = NumberFieldMapper.NumberType.DOUBLE.parse(value, false); } if (includeInAll) { diff --git a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 19f0936de540f..43636b6abd833 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -20,26 +20,26 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomPicks; - import org.apache.lucene.document.Document; import org.apache.lucene.document.FloatPoint; import org.apache.lucene.document.HalfFloatPoint; +import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.util.IOUtils; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType.Relation; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.junit.Before; import java.io.IOException; +import java.util.Arrays; public class NumberFieldTypeTests extends FieldTypeTestCase { @@ -62,6 +62,52 @@ public void testIsFieldWithinQuery() throws IOException { randomBoolean(), randomBoolean(), null, null, null)); } + public void testIntegerTermsQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null)); + assertEquals(IntPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null)); + assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery); + } + + public void testLongTermsQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1, 2.1), null)); + assertEquals(LongPoint.newSetQuery("field", 1), ft.termsQuery(Arrays.asList(1.0, 2.1), null)); + assertTrue(ft.termsQuery(Arrays.asList(1.1, 2.1), null) instanceof MatchNoDocsQuery); + } + + public void testByteTermQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery); + } + + public void testShortTermQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery); + } + + public void testIntegerTermQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery); + } + + public void testLongTermQueryWithDecimalPart() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertTrue(ft.termQuery(42.1, null) instanceof MatchNoDocsQuery); + } + public void testTermQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); ft.setName("field"); @@ -74,6 +120,82 @@ public void testTermQuery() { assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } + public void testRangeQueryWithNegativeBounds() { + MappedFieldType ftInt = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER); + ftInt.setName("field"); + ftInt.setIndexOptions(IndexOptions.DOCS); + assertEquals(IntPoint.newRangeQuery("field", -3, -3), ftInt.rangeQuery(-3.5, -2.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", -3, -3), ftInt.rangeQuery(-3.5, -2.5, false, false, null)); + assertEquals(IntPoint.newRangeQuery("field", 0, 0), ftInt.rangeQuery(-0.5, 0.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 0, 0), ftInt.rangeQuery(-0.5, 0.5, false, false, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 2), ftInt.rangeQuery(0.5, 2.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 2), ftInt.rangeQuery(0.5, 2.5, false, false, null)); + assertEquals(IntPoint.newRangeQuery("field", 0, 2), ftInt.rangeQuery(-0.5, 2.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 0, 2), ftInt.rangeQuery(-0.5, 2.5, false, false, null)); + + assertEquals(IntPoint.newRangeQuery("field", -2, 0), ftInt.rangeQuery(-2.5, 0.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", -2, 0), ftInt.rangeQuery(-2.5, 0.5, false, false, null)); + assertEquals(IntPoint.newRangeQuery("field", -2, -1), ftInt.rangeQuery(-2.5, -0.5, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", -2, -1), ftInt.rangeQuery(-2.5, -0.5, false, false, null)); + + MappedFieldType ftLong = new NumberFieldMapper.NumberFieldType(NumberType.LONG); + ftLong.setName("field"); + ftLong.setIndexOptions(IndexOptions.DOCS); + assertEquals(LongPoint.newRangeQuery("field", -3, -3), ftLong.rangeQuery(-3.5, -2.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", -3, -3), ftLong.rangeQuery(-3.5, -2.5, false, false, null)); + assertEquals(LongPoint.newRangeQuery("field", 0, 0), ftLong.rangeQuery(-0.5, 0.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 0, 0), ftLong.rangeQuery(-0.5, 0.5, false, false, null)); + assertEquals(LongPoint.newRangeQuery("field", 1, 2), ftLong.rangeQuery(0.5, 2.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 1, 2), ftLong.rangeQuery(0.5, 2.5, false, false, null)); + assertEquals(LongPoint.newRangeQuery("field", 0, 2), ftLong.rangeQuery(-0.5, 2.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 0, 2), ftLong.rangeQuery(-0.5, 2.5, false, false, null)); + + assertEquals(LongPoint.newRangeQuery("field", -2, 0), ftLong.rangeQuery(-2.5, 0.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", -2, 0), ftLong.rangeQuery(-2.5, 0.5, false, false, null)); + assertEquals(LongPoint.newRangeQuery("field", -2, -1), ftLong.rangeQuery(-2.5, -0.5, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", -2, -1), ftLong.rangeQuery(-2.5, -0.5, false, false, null)); + } + + public void testByteRangeQueryWithDecimalParts() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.BYTE); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null)); + } + + public void testShortRangeQueryWithDecimalParts() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.SHORT); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null)); + } + + public void testIntegerRangeQueryWithDecimalParts() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.INTEGER); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null)); + assertEquals(IntPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null)); + } + + public void testLongRangeQueryWithDecimalParts() { + MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberType.LONG); + ft.setName("field"); + ft.setIndexOptions(IndexOptions.DOCS); + assertEquals(LongPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 2, 10), ft.rangeQuery(1.1, 10, false, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, true, null)); + assertEquals(LongPoint.newRangeQuery("field", 1, 10), ft.rangeQuery(1, 10.1, true, false, null)); + } + public void testRangeQuery() { MappedFieldType ft = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); ft.setName("field"); @@ -87,36 +209,33 @@ public void testRangeQuery() { } public void testConversions() { - assertEquals((byte) 3, NumberType.BYTE.parse(3d)); - assertEquals((short) 3, NumberType.SHORT.parse(3d)); - assertEquals(3, NumberType.INTEGER.parse(3d)); - assertEquals(3L, NumberType.LONG.parse(3d)); - assertEquals(3f, NumberType.HALF_FLOAT.parse(3d)); - assertEquals(3f, NumberType.FLOAT.parse(3d)); - assertEquals(3d, NumberType.DOUBLE.parse(3d)); - - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(3.5)); - assertEquals("Value [3.5] has a decimal part", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(3.5)); - assertEquals("Value [3.5] has a decimal part", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(3.5)); - assertEquals("Value [3.5] has a decimal part", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(3.5)); - assertEquals("Value [3.5] has a decimal part", e.getMessage()); - assertEquals(3.5f, NumberType.FLOAT.parse(3.5)); - assertEquals(3.5d, NumberType.DOUBLE.parse(3.5)); - - e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128)); + assertEquals((byte) 3, NumberType.BYTE.parse(3d, true)); + assertEquals((short) 3, NumberType.SHORT.parse(3d, true)); + assertEquals(3, NumberType.INTEGER.parse(3d, true)); + assertEquals(3L, NumberType.LONG.parse(3d, true)); + assertEquals(3f, NumberType.HALF_FLOAT.parse(3d, true)); + assertEquals(3f, NumberType.FLOAT.parse(3d, true)); + assertEquals(3d, NumberType.DOUBLE.parse(3d, true)); + + assertEquals((byte) 3, NumberType.BYTE.parse(3.5, true)); + assertEquals((short) 3, NumberType.SHORT.parse(3.5, true)); + assertEquals(3, NumberType.INTEGER.parse(3.5, true)); + assertEquals(3L, NumberType.LONG.parse(3.5, true)); + + assertEquals(3.5f, NumberType.FLOAT.parse(3.5, true)); + assertEquals(3.5d, NumberType.DOUBLE.parse(3.5, true)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> NumberType.BYTE.parse(128, true)); assertEquals("Value [128] is out of range for a byte", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536)); + e = expectThrows(IllegalArgumentException.class, () -> NumberType.SHORT.parse(65536, true)); assertEquals("Value [65536] is out of range for a short", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L)); + e = expectThrows(IllegalArgumentException.class, () -> NumberType.INTEGER.parse(2147483648L, true)); assertEquals("Value [2147483648] is out of range for an integer", e.getMessage()); - e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d)); + e = expectThrows(IllegalArgumentException.class, () -> NumberType.LONG.parse(10000000000000000000d, true)); assertEquals("Value [1.0E19] is out of range for a long", e.getMessage()); - assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1)); - assertEquals(1.1f, NumberType.FLOAT.parse(1.1)); - assertEquals(1.1d, NumberType.DOUBLE.parse(1.1)); + assertEquals(1.1f, NumberType.HALF_FLOAT.parse(1.1, true)); + assertEquals(1.1f, NumberType.FLOAT.parse(1.1, true)); + assertEquals(1.1d, NumberType.DOUBLE.parse(1.1, true)); } public void testHalfFloatRange() throws IOException { From f5f2149ff2c66cbe1d09c57d34fd56ae6d298272 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Dec 2016 11:00:34 -0500 Subject: [PATCH 17/22] Remove much ceremony from parsing client yaml test suites (#22311) * Remove a checked exception, replacing it with `ParsingException`. * Remove all Parser classes for the yaml sections, replacing them with static methods. * Remove `ClientYamlTestFragmentParser`. Isn't used any more. * Remove `ClientYamlTestSuiteParseContext`, replacing it with some static utility methods. I did not rewrite the parsers using `ObjectParser` because I don't think it is worth it right now. --- .../ExceptionSerializationTests.java | 1 - .../test/rest/DebClientYamlTestSuiteIT.java | 3 +- .../IntegTestZipClientYamlTestSuiteIT.java | 3 +- .../test/rest/RpmClientYamlTestSuiteIT.java | 3 +- .../test/rest/TarClientYamlTestSuiteIT.java | 3 +- .../test/rest/ZipClientYamlTestSuiteIT.java | 3 +- .../smoketest/DocsClientYamlTestSuiteIT.java | 3 +- .../MatrixStatsClientYamlTestSuiteIT.java | 3 +- .../IngestCommonClientYamlTestSuiteIT.java | 3 +- .../LangExpressionClientYamlTestSuiteIT.java | 3 +- .../LangMustacheClientYamlTestSuiteIT.java | 3 +- .../LangPainlessClientYamlTestSuiteIT.java | 3 +- .../PercolatorClientYamlTestSuiteIT.java | 3 +- .../reindex/ReindexClientYamlTestSuiteIT.java | 3 +- .../netty4/Netty4ClientYamlTestSuiteIT.java | 3 +- .../analysis/IcuClientYamlTestSuiteIT.java | 3 +- .../KuromojiClientYamlTestSuiteIT.java | 3 +- .../PhoneticClientYamlTestSuiteIT.java | 3 +- .../SmartCNClientYamlTestSuiteIT.java | 3 +- .../StempelClientYamlTestSuiteIT.java | 3 +- .../UkrainianClientYamlTestSuiteIT.java | 3 +- ...veryAzureClassicClientYamlTestSuiteIT.java | 3 +- .../aws/CloudAwsClientYamlTestSuiteIT.java | 3 +- ...leBasedDiscoveryClientYamlTestSuiteIT.java | 4 +- .../DiscoveryGceClientYamlTestSuiteIT.java | 3 +- ...IngestAttachmentClientYamlTestSuiteIT.java | 3 +- .../IngestGeoIpClientYamlTestSuiteIT.java | 3 +- .../IngestUserAgentClientYamlTestSuiteIT.java | 3 +- .../JvmExampleClientYamlTestSuiteIT.java | 3 +- .../MapperMurmur3ClientYamlTestSuiteIT.java | 3 +- .../size/MapperSizeClientYamlTestSuiteIT.java | 3 +- .../RepositoryAzureClientYamlTestSuiteIT.java | 3 +- .../RepositoryGcsClientYamlTestSuiteIT.java | 3 +- .../RepositoryHdfsClientYamlTestSuiteIT.java | 7 +- .../s3/RepositoryS3ClientYamlTestSuiteIT.java | 3 +- .../store/StoreSmbClientYamlTestSuiteIT.java | 3 +- .../Backwards50ClientYamlTestSuiteIT.java | 4 +- .../UpgradeClusterClientYamlTestSuiteIT.java | 4 +- ...stIngestDisabledClientYamlTestSuiteIT.java | 3 +- ...ngestWithAllDepsClientYamlTestSuiteIT.java | 3 +- ...okeTestMultiNodeClientYamlTestSuiteIT.java | 5 +- ...SmokeTestPluginsClientYamlTestSuiteIT.java | 3 +- ...ndexWithPainlessClientYamlTestSuiteIT.java | 3 +- .../tribe/TribeClientYamlTestSuiteIT.java | 4 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 10 +- .../parser/ClientYamlTestFragmentParser.java | 33 -- .../parser/ClientYamlTestParseException.java | 33 -- .../parser/ClientYamlTestSectionParser.java | 56 --- .../ClientYamlTestSuiteParseContext.java | 195 -------- .../parser/ClientYamlTestSuiteParser.java | 101 ---- .../rest/yaml/parser/DoSectionParser.java | 125 ----- .../yaml/parser/GreaterThanEqualToParser.java | 43 -- .../rest/yaml/parser/GreaterThanParser.java | 42 -- .../test/rest/yaml/parser/IsFalseParser.java | 34 -- .../test/rest/yaml/parser/IsTrueParser.java | 34 -- .../test/rest/yaml/parser/LengthParser.java | 50 -- .../yaml/parser/LessThanOrEqualToParser.java | 43 -- .../test/rest/yaml/parser/LessThanParser.java | 42 -- .../test/rest/yaml/parser/MatchParser.java | 38 -- .../rest/yaml/parser/SetSectionParser.java | 57 --- .../rest/yaml/parser/SetupSectionParser.java | 54 --- .../rest/yaml/parser/SkipSectionParser.java | 80 --- .../yaml/parser/TeardownSectionParser.java | 53 -- .../yaml/section/ClientYamlTestSection.java | 36 +- .../yaml/section/ClientYamlTestSuite.java | 67 +++ .../test/rest/yaml/section/DoSection.java | 87 ++++ .../rest/yaml/section/ExecutableSection.java | 32 ++ .../yaml/section/GreaterThanAssertion.java | 13 + .../section/GreaterThanEqualToAssertion.java | 13 + .../rest/yaml/section/IsFalseAssertion.java | 7 +- .../rest/yaml/section/IsTrueAssertion.java | 6 + .../rest/yaml/section/LengthAssertion.java | 19 + .../rest/yaml/section/LessThanAssertion.java | 14 +- .../section/LessThanOrEqualToAssertion.java | 13 + .../rest/yaml/section/MatchAssertion.java | 8 + .../test/rest/yaml/section/ParserUtils.java | 76 +++ .../test/rest/yaml/section/SetSection.java | 24 + .../test/rest/yaml/section/SetupSection.java | 38 ++ .../test/rest/yaml/section/SkipSection.java | 61 +++ .../rest/yaml/section/TeardownSection.java | 39 ++ .../yaml/parser/DoSectionParserTests.java | 455 ------------------ .../yaml/parser/SkipSectionParserTests.java | 144 ------ .../yaml/parser/TestSectionParserTests.java | 249 ---------- .../ClientYamlSuiteRestApiParserTests.java | 2 +- ...tClientYamlTestFragmentParserTestCase.java | 16 +- .../AssertionTests.java} | 33 +- .../section/ClientYamlTestSectionTests.java | 220 ++++++++- .../ClientYamlTestSuiteTests.java} | 55 +-- .../rest/yaml/section/DoSectionTests.java | 405 +++++++++++++++- .../SetSectionTests.java} | 25 +- .../SetupSectionTests.java} | 10 +- .../rest/yaml/section/SkipSectionTests.java | 99 +++- .../TeardownSectionTests.java} | 18 +- 93 files changed, 1373 insertions(+), 2174 deletions(-) delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java create mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java delete mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java delete mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java delete mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser => section}/AbstractClientYamlTestFragmentParserTestCase.java (69%) rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser/AssertionParsersTests.java => section/AssertionTests.java} (75%) rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser/ClientYamlSuiteTestParserTests.java => section/ClientYamlTestSuiteTests.java} (90%) rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser/SetSectionParserTests.java => section/SetSectionTests.java} (69%) rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser/SetupSectionParserTests.java => section/SetupSectionTests.java} (88%) rename test/framework/src/test/java/org/elasticsearch/test/rest/yaml/{parser/TeardownSectionParserTests.java => section/TeardownSectionTests.java} (80%) diff --git a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java index 786426822dcbc..f7660e3a2e695 100644 --- a/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java +++ b/core/src/test/java/org/elasticsearch/ExceptionSerializationTests.java @@ -121,7 +121,6 @@ public void testExceptionRegistration() final Path startPath = PathUtils.get(ElasticsearchException.class.getProtectionDomain().getCodeSource().getLocation().toURI()) .resolve("org").resolve("elasticsearch"); final Set> ignore = Sets.newHashSet( - org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException.class, CancellableThreadsTests.CustomException.class, org.elasticsearch.rest.BytesRestResponseTests.WithHeadersException.class, AbstractClientHeadersTestCase.InternalException.class); diff --git a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java index a63b304a1d4e5..9a7978f69bfa9 100644 --- a/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java +++ b/distribution/deb/src/test/java/org/elasticsearch/test/rest/DebClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public DebClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java index c81ff7439f9bd..c3a038392cbe3 100644 --- a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java +++ b/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public IntegTestZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java index 9569dfe4d1066..86b4932180824 100644 --- a/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java +++ b/distribution/rpm/src/test/java/org/elasticsearch/test/rest/RpmClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public RpmClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java index 0c811c383d0f8..a86e398830cf1 100644 --- a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java +++ b/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public TarClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java index 52581c8e765c4..93c31c4ab6566 100644 --- a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java +++ b/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public ZipClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java index 87ca5acf1ca0d..a79730339e339 100644 --- a/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java +++ b/docs/src/test/java/org/elasticsearch/smoketest/DocsClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; import java.util.List; @@ -36,7 +35,7 @@ public DocsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandi } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } diff --git a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java index 11ddd2dfd4197..6e11a56d2c671 100644 --- a/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java +++ b/modules/aggs-matrix-stats/src/test/java/org/elasticsearch/search/aggregations/matrix/MatrixStatsClientYamlTestSuiteIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -33,7 +32,7 @@ public MatrixStatsClientYamlTestSuiteIT(@Name("yaml")ClientYamlTestCandidate tes } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java index 1b678835c4b58..4027a75a2d109 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/IngestCommonClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public IngestCommonClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java index 9a30def83e17d..a20050b80adcc 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/LangExpressionClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public LangExpressionClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java index 377fa870c41b0..72eb9f2ad79a6 100644 --- a/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java +++ b/modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/LangMustacheClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public LangMustacheClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java index 55d3f1c810176..9d055b74cc89f 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/LangPainlessClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -36,7 +35,7 @@ public LangPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java index db1bbf13c83fa..8efd350839837 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public PercolatorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java index 54483eae569a9..0975cad96a400 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public ReindexClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java index 237227cd4df3d..2341f3905efb4 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4ClientYamlTestSuiteIT.java @@ -26,7 +26,6 @@ import org.apache.lucene.util.TimeUnits; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -39,7 +38,7 @@ public Netty4ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCan } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java index ce2e660ecfcd4..04fc222063c1e 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/analysis/IcuClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public IcuClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandid } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java index 0797b10d7743b..bc5360f94d1d5 100644 --- a/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java +++ b/plugins/analysis-kuromoji/src/test/java/org/elasticsearch/index/analysis/KuromojiClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public KuromojiClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java index 447eb1d6cd772..3d892bddee44f 100644 --- a/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java +++ b/plugins/analysis-phonetic/src/test/java/org/elasticsearch/index/analysis/PhoneticClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public PhoneticClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java index 534af79a199c4..24a581d9e26bd 100644 --- a/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java +++ b/plugins/analysis-smartcn/src/test/java/org/elasticsearch/index/analysis/SmartCNClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public SmartCNClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java index 56edcdb692cb7..371431e1c2567 100644 --- a/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java +++ b/plugins/analysis-stempel/src/test/java/org/elasticsearch/index/analysis/StempelClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public StempelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCa } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java index dd77fdf74a543..50d935e5228b0 100644 --- a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate test } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java index 33c5d41f70c76..4f6a44ef2357f 100644 --- a/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java +++ b/plugins/discovery-azure-classic/src/test/java/org/elasticsearch/discovery/azure/classic/DiscoveryAzureClassicClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public DiscoveryAzureClassicClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCa } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java index 3cd30c187d4f1..3488b5ea49c9b 100644 --- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java +++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/cloud/aws/CloudAwsClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public CloudAwsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java index 8a0bd808dbabd..dbc6212af9fff 100644 --- a/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java +++ b/plugins/discovery-file/src/test/java/org/elasticsearch/discovery/file/FileBasedDiscoveryClientYamlTestSuiteIT.java @@ -21,9 +21,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -37,7 +37,7 @@ public FileBasedDiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandi } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java index 3af39b6da5d0c..67ef297609f95 100644 --- a/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java +++ b/plugins/discovery-gce/src/test/java/org/elasticsearch/discovery/gce/DiscoveryGceClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public DiscoveryGceClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java index 40e95451e4936..43e9cd394f0aa 100644 --- a/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java +++ b/plugins/ingest-attachment/src/test/java/org/elasticsearch/ingest/attachment/IngestAttachmentClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public IngestAttachmentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandida } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java index ed381dab0b672..33f7c67e018fb 100644 --- a/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java +++ b/plugins/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/IngestGeoIpClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public IngestGeoIpClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate te } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java index 2acac87363721..cc70c1ef6ba6f 100644 --- a/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java +++ b/plugins/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/IngestUserAgentClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public IngestUserAgentClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidat } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java index b7bae90817b09..d3e84316b1441 100644 --- a/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java +++ b/plugins/jvm-example/src/test/java/org/elasticsearch/plugin/example/JvmExampleClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public JvmExampleClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java index 3e9a5f139273e..9ca9f677b8d97 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/MapperMurmur3ClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public MapperMurmur3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java index d8de3635b77d8..3741c5626f608 100644 --- a/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java +++ b/plugins/mapper-size/src/test/java/org/elasticsearch/index/mapper/size/MapperSizeClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public MapperSizeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java index 2919d0739780b..5d3d051fb3be0 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/RepositoryAzureClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public RepositoryAzureClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidat } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java index 6ed036e277f63..cdd86bc48b990 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/RepositoryGcsClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public RepositoryGcsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java index 264a350d5141a..6e66b0d49fee2 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/RepositoryHdfsClientYamlTestSuiteIT.java @@ -18,14 +18,13 @@ */ package org.elasticsearch.repositories.hdfs; -import java.io.IOException; - import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; + +import java.io.IOException; public class RepositoryHdfsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { @@ -34,7 +33,7 @@ public RepositoryHdfsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java index 9c567a570fe7f..30056f67d2f99 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public RepositoryS3ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate t } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java index 0b9de745cac39..0216083c95f61 100644 --- a/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java +++ b/plugins/store-smb/src/test/java/org/elasticsearch/index/store/StoreSmbClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public StoreSmbClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testC } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java index af77b216bc47e..cac063fce7837 100644 --- a/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java +++ b/qa/backwards-5.0/src/test/java/org/elasticsearch/backwards/Backwards50ClientYamlTestSuiteIT.java @@ -21,10 +21,10 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + import org.apache.lucene.util.TimeUnits; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -36,7 +36,7 @@ public Backwards50ClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate) { } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java index f58d618adf379..b7b825da47240 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeClusterClientYamlTestSuiteIT.java @@ -21,10 +21,10 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + import org.apache.lucene.util.TimeUnits; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -41,7 +41,7 @@ public UpgradeClusterClientYamlTestSuiteIT(ClientYamlTestCandidate testCandidate } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } } diff --git a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java index beb7499bf7fb3..7f4c2c4a4d523 100644 --- a/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-disabled/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestDisabledClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public SmokeTestIngestDisabledClientYamlTestSuiteIT(@Name("yaml") ClientYamlTest } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java index 0bd7b9ac029b1..d1e1adabfd030 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-ingest-with-all-dependencies/src/test/java/org/elasticsearch/smoketest/SmokeTestIngestWithAllDepsClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public SmokeTestIngestWithAllDepsClientYamlTestSuiteIT(@Name("yaml") ClientYamlT } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } diff --git a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java index 225e12f65fda3..69abd0b3cc2bc 100644 --- a/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-multinode/src/test/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java @@ -21,12 +21,11 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; - import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + import org.apache.lucene.util.TimeUnits; import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -38,7 +37,7 @@ public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandi } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java index 0f5636095e979..2ae7be8fb1eb8 100644 --- a/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java +++ b/qa/smoke-test-plugins/src/test/java/org/elasticsearch/smoketest/SmokeTestPluginsClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -35,7 +34,7 @@ public SmokeTestPluginsClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandida } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java index fb4f27210d83a..5366eaf4bd13e 100644 --- a/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java +++ b/qa/smoke-test-reindex-with-painless/src/test/java/org/elasticsearch/smoketest/SmokeTestReindexWithPainlessClientYamlTestSuiteIT.java @@ -24,7 +24,6 @@ import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -34,7 +33,7 @@ public SmokeTestReindexWithPainlessClientYamlTestSuiteIT(@Name("yaml") ClientYam } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return ESClientYamlSuiteTestCase.createParameters(); } } diff --git a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java index 6013913bdc4d1..211043ed4b064 100644 --- a/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java +++ b/qa/smoke-test-tribe-node/src/test/java/org/elasticsearch/tribe/TribeClientYamlTestSuiteIT.java @@ -21,9 +21,9 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; import java.io.IOException; @@ -46,7 +46,7 @@ public TribeClientYamlTestSuiteIT(@Name("yaml") final ClientYamlTestCandidate te } @ParametersFactory - public static Iterable parameters() throws IOException, ClientYamlTestParseException { + public static Iterable parameters() throws IOException { return createParameters(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index e4434d1365b5f..f025056a4e316 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml; import com.carrotsearch.randomizedtesting.RandomizedTest; + import org.apache.http.HttpHost; import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; @@ -31,8 +32,6 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.rest.ESRestTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; @@ -186,7 +185,7 @@ protected void afterIfFailed(List errors) { super.afterIfFailed(errors); } - public static Iterable createParameters() throws IOException, ClientYamlTestParseException { + public static Iterable createParameters() throws IOException { List restTestCandidates = collectTestCandidates(); List objects = new ArrayList<>(); for (ClientYamlTestCandidate restTestCandidate : restTestCandidates) { @@ -195,7 +194,7 @@ public static Iterable createParameters() throws IOException, ClientYa return objects; } - private static List collectTestCandidates() throws ClientYamlTestParseException, IOException { + private static List collectTestCandidates() throws IOException { List testCandidates = new ArrayList<>(); FileSystem fileSystem = getFileSystem(); // don't make a try-with, getFileSystem returns null @@ -203,12 +202,11 @@ private static List collectTestCandidates() throws Clie try { String[] paths = resolvePathsProperty(REST_TESTS_SUITE, DEFAULT_TESTS_PATH); Map> yamlSuites = FileUtils.findYamlSuites(fileSystem, DEFAULT_TESTS_PATH, paths); - ClientYamlTestSuiteParser restTestSuiteParser = new ClientYamlTestSuiteParser(); //yaml suites are grouped by directory (effectively by api) for (String api : yamlSuites.keySet()) { List yamlFiles = new ArrayList<>(yamlSuites.get(api)); for (Path yamlFile : yamlFiles) { - ClientYamlTestSuite restTestSuite = restTestSuiteParser.parse(api, yamlFile); + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(api, yamlFile); for (ClientYamlTestSection testSection : restTestSuite.getTestSections()) { testCandidates.add(new ClientYamlTestCandidate(restTestSuite, testSection)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java deleted file mode 100644 index 390ac1ce366c6..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestFragmentParser.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import java.io.IOException; - -/** - * Base parser for a REST test suite fragment - * @param the test fragment's type that gets parsed and returned - */ -public interface ClientYamlTestFragmentParser { - - /** - * Parses a test fragment given the current {@link ClientYamlTestSuiteParseContext} - */ - T parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException; -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java deleted file mode 100644 index 594f701c79ab7..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestParseException.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -/** - * Exception thrown whenever there is a problem parsing any of the REST test suite fragment - */ -public class ClientYamlTestParseException extends Exception { - - ClientYamlTestParseException(String message) { - super(message); - } - - ClientYamlTestParseException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java deleted file mode 100644 index b6b6adfd037c0..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSectionParser.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; - -import java.io.IOException; - -/** - * Parser for a complete test section - */ -public class ClientYamlTestSectionParser implements ClientYamlTestFragmentParser { - - @Override - public ClientYamlTestSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentParser parser = parseContext.parser(); - parseContext.advanceToFieldName(); - ClientYamlTestSection testSection = new ClientYamlTestSection(parser.currentName()); - try { - parser.nextToken(); - testSection.setSkipSection(parseContext.parseSkipSection()); - - while ( parser.currentToken() != XContentParser.Token.END_ARRAY) { - parseContext.advanceToFieldName(); - testSection.addExecutableSection(parseContext.parseExecutableSection()); - } - - parser.nextToken(); - assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected " - + XContentParser.Token.END_OBJECT + " but was " + parser.currentToken(); - parser.nextToken(); - - return testSection; - } catch (Exception e) { - throw new ClientYamlTestParseException("Error parsing test named [" + testSection.getName() + "]", e); - } - } - -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java deleted file mode 100644 index e466c1010f6b2..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParseContext.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.ParseFieldMatcher; -import org.elasticsearch.common.ParseFieldMatcherSupplier; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.ExecutableSection; -import org.elasticsearch.test.rest.yaml.section.SetupSection; -import org.elasticsearch.test.rest.yaml.section.SkipSection; -import org.elasticsearch.test.rest.yaml.section.TeardownSection; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Context shared across the whole tests parse phase. - * Provides shared parse methods and holds information needed to parse the test sections (e.g. es version) - */ -public class ClientYamlTestSuiteParseContext implements ParseFieldMatcherSupplier { - - private static final SetupSectionParser SETUP_SECTION_PARSER = new SetupSectionParser(); - private static final TeardownSectionParser TEARDOWN_SECTION_PARSER = new TeardownSectionParser(); - private static final ClientYamlTestSectionParser TEST_SECTION_PARSER = new ClientYamlTestSectionParser(); - private static final SkipSectionParser SKIP_SECTION_PARSER = new SkipSectionParser(); - private static final DoSectionParser DO_SECTION_PARSER = new DoSectionParser(); - private static final Map> EXECUTABLE_SECTIONS_PARSERS = - new HashMap<>(); - static { - EXECUTABLE_SECTIONS_PARSERS.put("do", DO_SECTION_PARSER); - EXECUTABLE_SECTIONS_PARSERS.put("set", new SetSectionParser()); - EXECUTABLE_SECTIONS_PARSERS.put("match", new MatchParser()); - EXECUTABLE_SECTIONS_PARSERS.put("is_true", new IsTrueParser()); - EXECUTABLE_SECTIONS_PARSERS.put("is_false", new IsFalseParser()); - EXECUTABLE_SECTIONS_PARSERS.put("gt", new GreaterThanParser()); - EXECUTABLE_SECTIONS_PARSERS.put("gte", new GreaterThanEqualToParser()); - EXECUTABLE_SECTIONS_PARSERS.put("lt", new LessThanParser()); - EXECUTABLE_SECTIONS_PARSERS.put("lte", new LessThanOrEqualToParser()); - EXECUTABLE_SECTIONS_PARSERS.put("length", new LengthParser()); - } - - private final String api; - private final String suiteName; - private final XContentParser parser; - - public ClientYamlTestSuiteParseContext(String api, String suiteName, XContentParser parser) { - this.api = api; - this.suiteName = suiteName; - this.parser = parser; - } - - public String getApi() { - return api; - } - - public String getSuiteName() { - return suiteName; - } - - public XContentParser parser() { - return parser; - } - - public SetupSection parseSetupSection() throws IOException, ClientYamlTestParseException { - - advanceToFieldName(); - - if ("setup".equals(parser.currentName())) { - parser.nextToken(); - SetupSection setupSection = SETUP_SECTION_PARSER.parse(this); - parser.nextToken(); - return setupSection; - } - - return SetupSection.EMPTY; - } - - public TeardownSection parseTeardownSection() throws IOException, ClientYamlTestParseException { - advanceToFieldName(); - - if ("teardown".equals(parser.currentName())) { - parser.nextToken(); - TeardownSection teardownSection = TEARDOWN_SECTION_PARSER.parse(this); - parser.nextToken(); - return teardownSection; - } - - return TeardownSection.EMPTY; - } - - public ClientYamlTestSection parseTestSection() throws IOException, ClientYamlTestParseException { - return TEST_SECTION_PARSER.parse(this); - } - - public SkipSection parseSkipSection() throws IOException, ClientYamlTestParseException { - - advanceToFieldName(); - - if ("skip".equals(parser.currentName())) { - SkipSection skipSection = SKIP_SECTION_PARSER.parse(this); - parser.nextToken(); - return skipSection; - } - - return SkipSection.EMPTY; - } - - public ExecutableSection parseExecutableSection() throws IOException, ClientYamlTestParseException { - advanceToFieldName(); - String section = parser.currentName(); - ClientYamlTestFragmentParser execSectionParser = EXECUTABLE_SECTIONS_PARSERS.get(section); - if (execSectionParser == null) { - throw new ClientYamlTestParseException("no parser found for executable section [" + section + "]"); - } - XContentLocation location = parser.getTokenLocation(); - try { - ExecutableSection executableSection = execSectionParser.parse(this); - parser.nextToken(); - return executableSection; - } catch (Exception e) { - throw new IOException("Error parsing section starting at ["+ location + "]", e); - } - } - - public DoSection parseDoSection() throws IOException, ClientYamlTestParseException { - return DO_SECTION_PARSER.parse(this); - } - - public void advanceToFieldName() throws IOException, ClientYamlTestParseException { - XContentParser.Token token = parser.currentToken(); - //we are in the beginning, haven't called nextToken yet - if (token == null) { - token = parser.nextToken(); - } - if (token == XContentParser.Token.START_ARRAY) { - token = parser.nextToken(); - } - if (token == XContentParser.Token.START_OBJECT) { - token = parser.nextToken(); - } - if (token != XContentParser.Token.FIELD_NAME) { - throw new ClientYamlTestParseException("malformed test section: field name expected but found " + token + " at " - + parser.getTokenLocation()); - } - } - - public String parseField() throws IOException, ClientYamlTestParseException { - parser.nextToken(); - assert parser.currentToken().isValue(); - String field = parser.text(); - parser.nextToken(); - return field; - } - - public Tuple parseTuple() throws IOException, ClientYamlTestParseException { - parser.nextToken(); - advanceToFieldName(); - Map map = parser.map(); - assert parser.currentToken() == XContentParser.Token.END_OBJECT; - parser.nextToken(); - - if (map.size() != 1) { - throw new ClientYamlTestParseException("expected key value pair but found " + map.size() + " "); - } - - Map.Entry entry = map.entrySet().iterator().next(); - return Tuple.tuple(entry.getKey(), entry.getValue()); - } - - @Override - public ParseFieldMatcher getParseFieldMatcher() { - return ParseFieldMatcher.STRICT; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java deleted file mode 100644 index 3c09452fc8f03..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlTestSuiteParser.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -/** - * Parser for a complete test suite (yaml file) - */ -public class ClientYamlTestSuiteParser implements ClientYamlTestFragmentParser { - - public ClientYamlTestSuite parse(String api, Path file) throws IOException, ClientYamlTestParseException { - - if (!Files.isRegularFile(file)) { - throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file"); - } - - String filename = file.getFileName().toString(); - //remove the file extension - int i = filename.lastIndexOf('.'); - if (i > 0) { - filename = filename.substring(0, i); - } - - //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break. - try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { - ByteBuffer bb = ByteBuffer.wrap(new byte[1]); - channel.read(bb, channel.size() - 1); - if (bb.get(0) != 10) { - throw new ClientYamlTestParseException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)"); - } - } - - try (XContentParser parser = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY, Files.newInputStream(file))) { - ClientYamlTestSuiteParseContext testParseContext = new ClientYamlTestSuiteParseContext(api, filename, parser); - return parse(testParseContext); - } catch(Exception e) { - throw new ClientYamlTestParseException("Error parsing " + api + "/" + filename, e); - } - } - - @Override - public ClientYamlTestSuite parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentParser parser = parseContext.parser(); - - parser.nextToken(); - assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was " - + parser.currentToken(); - - ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(parseContext.getApi(), parseContext.getSuiteName()); - - restTestSuite.setSetupSection(parseContext.parseSetupSection()); - restTestSuite.setTeardownSection(parseContext.parseTeardownSection()); - - while(true) { - //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed - //we need to somehow distinguish between a null in the middle of a test ("---") - // and a null at the end of the file (at least two consecutive null tokens) - if(parser.currentToken() == null) { - if (parser.nextToken() == null) { - break; - } - } - - ClientYamlTestSection testSection = parseContext.parseTestSection(); - if (!restTestSuite.addTestSection(testSection)) { - throw new ClientYamlTestParseException("duplicate test section [" + testSection.getName() + "] found in [" - + restTestSuite.getPath() + "]"); - } - } - - return restTestSuite; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java deleted file mode 100644 index ba746c129b615..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParser.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.rest.yaml.section.ApiCallSection; -import org.elasticsearch.test.rest.yaml.section.DoSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.unmodifiableList; - -/** - * Parser for do sections - */ -public class DoSectionParser implements ClientYamlTestFragmentParser { - - @Override - public DoSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - - XContentParser parser = parseContext.parser(); - - String currentFieldName = null; - XContentParser.Token token; - - DoSection doSection = new DoSection(parseContext.parser().getTokenLocation()); - ApiCallSection apiCallSection = null; - Map headers = new HashMap<>(); - List expectedWarnings = new ArrayList<>(); - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if ("catch".equals(currentFieldName)) { - doSection.setCatch(parser.text()); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if ("warnings".equals(currentFieldName)) { - while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) { - expectedWarnings.add(parser.text()); - } - if (token != XContentParser.Token.END_ARRAY) { - throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]"); - } - } else { - throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]"); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if ("headers".equals(currentFieldName)) { - String headerName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - headerName = parser.currentName(); - } else if (token.isValue()) { - headers.put(headerName, parser.text()); - } - } - } else if (currentFieldName != null) { // must be part of API call then - apiCallSection = new ApiCallSection(currentFieldName); - String paramName = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - paramName = parser.currentName(); - } else if (token.isValue()) { - if ("body".equals(paramName)) { - String body = parser.text(); - XContentType bodyContentType = XContentFactory.xContentType(body); - XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser( - NamedXContentRegistry.EMPTY, body); - //multiple bodies are supported e.g. in case of bulk provided as a whole string - while(bodyParser.nextToken() != null) { - apiCallSection.addBody(bodyParser.mapOrdered()); - } - } else { - apiCallSection.addParam(paramName, parser.text()); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if ("body".equals(paramName)) { - apiCallSection.addBody(parser.mapOrdered()); - } - } - } - } - } - } - try { - if (apiCallSection == null) { - throw new ClientYamlTestParseException("client call section is mandatory within a do section"); - } - if (headers.isEmpty() == false) { - apiCallSection.addHeaders(headers); - } - doSection.setApiCallSection(apiCallSection); - doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings)); - } finally { - parser.nextToken(); - } - return doSection; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java deleted file mode 100644 index 65a46d139bd06..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanEqualToParser.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.GreaterThanEqualToAssertion; - -import java.io.IOException; - -/** - * Parser for gte assert sections - */ -public class GreaterThanEqualToParser implements ClientYamlTestFragmentParser { - @Override - public GreaterThanEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) - throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new ClientYamlTestParseException("gte section can only be used with objects that support natural ordering, found " - + stringObjectTuple.v2().getClass().getSimpleName()); - } - return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java deleted file mode 100644 index f6c532720672c..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/GreaterThanParser.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion; - -import java.io.IOException; - -/** - * Parser for gt assert sections - */ -public class GreaterThanParser implements ClientYamlTestFragmentParser { - - @Override - public GreaterThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new ClientYamlTestParseException("gt section can only be used with objects that support natural ordering, found " - + stringObjectTuple.v2().getClass().getSimpleName()); - } - return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java deleted file mode 100644 index 3d9593ce29023..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsFalseParser.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion; - -import java.io.IOException; - -/** - * Parser for is_false assert sections - */ -public class IsFalseParser implements ClientYamlTestFragmentParser { - - @Override - public IsFalseAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - return new IsFalseAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java deleted file mode 100644 index 9dfc3be57e37b..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/IsTrueParser.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion; - -import java.io.IOException; - -/** - * Parser for is_true assert sections - */ -public class IsTrueParser implements ClientYamlTestFragmentParser { - - @Override - public IsTrueAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - return new IsTrueAssertion(parseContext.parser().getTokenLocation(), parseContext.parseField()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java deleted file mode 100644 index 98c030cd6a13c..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LengthParser.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.LengthAssertion; - -import java.io.IOException; - -/** - * Parser for length assert sections - */ -public class LengthParser implements ClientYamlTestFragmentParser { - - @Override - public LengthAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - assert stringObjectTuple.v2() != null; - int value; - if (stringObjectTuple.v2() instanceof Number) { - value = ((Number) stringObjectTuple.v2()).intValue(); - } else { - try { - value = Integer.valueOf(stringObjectTuple.v2().toString()); - } catch(NumberFormatException e) { - throw new ClientYamlTestParseException("length is not a valid number", e); - } - - } - return new LengthAssertion(location, stringObjectTuple.v1(), value); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java deleted file mode 100644 index 5dfe4c742f525..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanOrEqualToParser.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.LessThanOrEqualToAssertion; - -import java.io.IOException; - -/** - * Parser for lte assert section - */ -public class LessThanOrEqualToParser implements ClientYamlTestFragmentParser { - - @Override - public LessThanOrEqualToAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new ClientYamlTestParseException("lte section can only be used with objects that support natural ordering, found " - + stringObjectTuple.v2().getClass().getSimpleName()); - } - return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java deleted file mode 100644 index 0caf12ea91a02..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/LessThanParser.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.LessThanAssertion; - -import java.io.IOException; - -/** - * Parser for lt assert sections - */ -public class LessThanParser implements ClientYamlTestFragmentParser { - - @Override - public LessThanAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - if (! (stringObjectTuple.v2() instanceof Comparable) ) { - throw new ClientYamlTestParseException("lt section can only be used with objects that support natural ordering, found " - + stringObjectTuple.v2().getClass().getSimpleName()); - } - return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java deleted file mode 100644 index f456c32d0925d..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/MatchParser.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.rest.yaml.section.MatchAssertion; - -import java.io.IOException; - -/** - * Parser for match assert sections - */ -public class MatchParser implements ClientYamlTestFragmentParser { - - @Override - public MatchAssertion parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentLocation location = parseContext.parser().getTokenLocation(); - Tuple stringObjectTuple = parseContext.parseTuple(); - return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java deleted file mode 100644 index c7797e42d5787..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParser.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.SetSection; - -import java.io.IOException; - -/** - * Parser for set sections - */ -public class SetSectionParser implements ClientYamlTestFragmentParser { - - @Override - public SetSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - - XContentParser parser = parseContext.parser(); - - String currentFieldName = null; - XContentParser.Token token; - - SetSection setSection = new SetSection(parser.getTokenLocation()); - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - setSection.addSet(currentFieldName, parser.text()); - } - } - - parser.nextToken(); - - if (setSection.getStash().isEmpty()) { - throw new ClientYamlTestParseException("set section must set at least a value"); - } - - return setSection; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java deleted file mode 100644 index e62b3af5251e0..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParser.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.SetupSection; - -import java.io.IOException; - -/** - * Parser for setup sections - */ -public class SetupSectionParser implements ClientYamlTestFragmentParser { - - @Override - public SetupSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - - XContentParser parser = parseContext.parser(); - - SetupSection setupSection = new SetupSection(); - setupSection.setSkipSection(parseContext.parseSkipSection()); - - while (parser.currentToken() != XContentParser.Token.END_ARRAY) { - parseContext.advanceToFieldName(); - if (!"do".equals(parser.currentName())) { - throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within setup section"); - } - - parser.nextToken(); - setupSection.addDoSection(parseContext.parseDoSection()); - parser.nextToken(); - } - - parser.nextToken(); - - return setupSection; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java deleted file mode 100644 index b73edf7d2c6b5..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParser.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.SkipSection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Parser for skip sections - */ -public class SkipSectionParser implements ClientYamlTestFragmentParser { - - @Override - public SkipSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - - XContentParser parser = parseContext.parser(); - - String currentFieldName = null; - XContentParser.Token token; - String version = null; - String reason = null; - List features = new ArrayList<>(); - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token.isValue()) { - if ("version".equals(currentFieldName)) { - version = parser.text(); - } else if ("reason".equals(currentFieldName)) { - reason = parser.text(); - } else if ("features".equals(currentFieldName)) { - features.add(parser.text()); - } - else { - throw new ClientYamlTestParseException("field " + currentFieldName + " not supported within skip section"); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if ("features".equals(currentFieldName)) { - while(parser.nextToken() != XContentParser.Token.END_ARRAY) { - features.add(parser.text()); - } - } - } - } - - parser.nextToken(); - - if (!Strings.hasLength(version) && features.isEmpty()) { - throw new ClientYamlTestParseException("version or features is mandatory within skip section"); - } - if (Strings.hasLength(version) && !Strings.hasLength(reason)) { - throw new ClientYamlTestParseException("reason is mandatory within skip version section"); - } - - return new SkipSection(version, features, reason); - - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java deleted file mode 100644 index ed1b42c3a9cc7..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParser.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.rest.yaml.section.TeardownSection; - -import java.io.IOException; - -/** - * Parser for teardown section - */ -public class TeardownSectionParser implements ClientYamlTestFragmentParser { - - @Override - public TeardownSection parse(ClientYamlTestSuiteParseContext parseContext) throws IOException, ClientYamlTestParseException { - XContentParser parser = parseContext.parser(); - - TeardownSection teardownSection = new TeardownSection(); - teardownSection.setSkipSection(parseContext.parseSkipSection()); - - while (parser.currentToken() != XContentParser.Token.END_ARRAY) { - parseContext.advanceToFieldName(); - if (!"do".equals(parser.currentName())) { - throw new ClientYamlTestParseException("section [" + parser.currentName() + "] not supported within teardown section"); - } - - parser.nextToken(); - teardownSection.addDoSection(parseContext.parseDoSection()); - parser.nextToken(); - } - - parser.nextToken(); - return teardownSection; - } -} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index cd9cc1fbe488c..cbef40fbcb748 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -18,6 +18,11 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -25,15 +30,44 @@ * Represents a test section, which is composed of a skip section and multiple executable sections. */ public class ClientYamlTestSection implements Comparable { + public static ClientYamlTestSection parse(XContentParser parser) throws IOException { + ParserUtils.advanceToFieldName(parser); + ClientYamlTestSection testSection = new ClientYamlTestSection(parser.getTokenLocation(), parser.currentName()); + try { + parser.nextToken(); + testSection.setSkipSection(SkipSection.parseIfNext(parser)); + + while (parser.currentToken() != XContentParser.Token.END_ARRAY) { + ParserUtils.advanceToFieldName(parser); + testSection.addExecutableSection(ExecutableSection.parse(parser)); + } + + parser.nextToken(); + assert parser.currentToken() == XContentParser.Token.END_OBJECT : "malformed section [" + testSection.getName() + "] expected " + + XContentParser.Token.END_OBJECT + " but was " + parser.currentToken(); + parser.nextToken(); + + return testSection; + } catch (Exception e) { + throw new ParsingException(parser.getTokenLocation(), "Error parsing test named [" + testSection.getName() + "]", e); + } + } + + private final XContentLocation location; private final String name; private SkipSection skipSection; private final List executableSections; - public ClientYamlTestSection(String name) { + public ClientYamlTestSection(XContentLocation location, String name) { + this.location = location; this.name = name; this.executableSections = new ArrayList<>(); } + public XContentLocation getLocation() { + return location; + } + public String getName() { return name; } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java index af3e1be0528f3..5f97eb4af6a8f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java @@ -18,6 +18,16 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -28,6 +38,63 @@ * Supports a setup section and multiple test sections. */ public class ClientYamlTestSuite { + public static ClientYamlTestSuite parse(String api, Path file) throws IOException { + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException(file.toAbsolutePath() + " is not a file"); + } + + String filename = file.getFileName().toString(); + //remove the file extension + int i = filename.lastIndexOf('.'); + if (i > 0) { + filename = filename.substring(0, i); + } + + //our yaml parser seems to be too tolerant. Each yaml suite must end with \n, otherwise clients tests might break. + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.READ)) { + ByteBuffer bb = ByteBuffer.wrap(new byte[1]); + channel.read(bb, channel.size() - 1); + if (bb.get(0) != 10) { + throw new IOException("test suite [" + api + "/" + filename + "] doesn't end with line feed (\\n)"); + } + } + + try (XContentParser parser = YamlXContent.yamlXContent.createParser(ExecutableSection.XCONTENT_REGISTRY, + Files.newInputStream(file))) { + return parse(api, filename, parser); + } catch(Exception e) { + throw new IOException("Error parsing " + api + "/" + filename, e); + } + } + + public static ClientYamlTestSuite parse(String api, String suiteName, XContentParser parser) throws IOException { + parser.nextToken(); + assert parser.currentToken() == XContentParser.Token.START_OBJECT : "expected token to be START_OBJECT but was " + + parser.currentToken(); + + ClientYamlTestSuite restTestSuite = new ClientYamlTestSuite(api, suiteName); + + restTestSuite.setSetupSection(SetupSection.parseIfNext(parser)); + restTestSuite.setTeardownSection(TeardownSection.parseIfNext(parser)); + + while(true) { + //the "---" section separator is not understood by the yaml parser. null is returned, same as when the parser is closed + //we need to somehow distinguish between a null in the middle of a test ("---") + // and a null at the end of the file (at least two consecutive null tokens) + if(parser.currentToken() == null) { + if (parser.nextToken() == null) { + break; + } + } + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + if (!restTestSuite.addTestSection(testSection)) { + throw new ParsingException(testSection.getLocation(), "duplicate test section [" + testSection.getName() + "]"); + } + } + + return restTestSuite; + } private final String api; private final String name; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index e233e9fab8099..1729238040456 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -19,15 +19,21 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.elasticsearch.test.rest.yaml.ClientYamlTestResponseException; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; @@ -35,6 +41,7 @@ import java.util.Set; import static java.util.Collections.emptyList; +import static java.util.Collections.unmodifiableList; import static org.elasticsearch.common.collect.Tuple.tuple; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; import static org.hamcrest.Matchers.allOf; @@ -65,6 +72,86 @@ * */ public class DoSection implements ExecutableSection { + public static DoSection parse(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + + DoSection doSection = new DoSection(parser.getTokenLocation()); + ApiCallSection apiCallSection = null; + Map headers = new HashMap<>(); + List expectedWarnings = new ArrayList<>(); + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("catch".equals(currentFieldName)) { + doSection.setCatch(parser.text()); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("warnings".equals(currentFieldName)) { + while ((token = parser.nextToken()) == XContentParser.Token.VALUE_STRING) { + expectedWarnings.add(parser.text()); + } + if (token != XContentParser.Token.END_ARRAY) { + throw new ParsingException(parser.getTokenLocation(), "[warnings] must be a string array but saw [" + token + "]"); + } + } else { + throw new ParsingException(parser.getTokenLocation(), "unknown array [" + currentFieldName + "]"); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if ("headers".equals(currentFieldName)) { + String headerName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + headerName = parser.currentName(); + } else if (token.isValue()) { + headers.put(headerName, parser.text()); + } + } + } else if (currentFieldName != null) { // must be part of API call then + apiCallSection = new ApiCallSection(currentFieldName); + String paramName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + paramName = parser.currentName(); + } else if (token.isValue()) { + if ("body".equals(paramName)) { + String body = parser.text(); + XContentType bodyContentType = XContentFactory.xContentType(body); + XContentParser bodyParser = XContentFactory.xContent(bodyContentType).createParser( + NamedXContentRegistry.EMPTY, body); + //multiple bodies are supported e.g. in case of bulk provided as a whole string + while(bodyParser.nextToken() != null) { + apiCallSection.addBody(bodyParser.mapOrdered()); + } + } else { + apiCallSection.addParam(paramName, parser.text()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if ("body".equals(paramName)) { + apiCallSection.addBody(parser.mapOrdered()); + } + } + } + } + } + } + try { + if (apiCallSection == null) { + throw new IllegalArgumentException("client call section is mandatory within a do section"); + } + if (headers.isEmpty() == false) { + apiCallSection.addHeaders(headers); + } + doSection.setApiCallSection(apiCallSection); + doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings)); + } finally { + parser.nextToken(); + } + return doSection; + } + private static final Logger logger = Loggers.getLogger(DoSection.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java index 11b8f3e7a0f65..827457f4c2ae2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ExecutableSection.java @@ -18,15 +18,47 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; import java.io.IOException; +import java.util.Arrays; /** * Represents a test fragment that can be executed (e.g. api call, assertion) */ public interface ExecutableSection { + /** + * {@link NamedXContentRegistry} needed in the {@link XContentParser} before calling {@link ExecutableSection#parse(XContentParser)}. + */ + NamedXContentRegistry XCONTENT_REGISTRY = new NamedXContentRegistry(Arrays.asList( + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("do"), DoSection::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("set"), SetSection::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("match"), MatchAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_true"), IsTrueAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("is_false"), IsFalseAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gt"), GreaterThanAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("gte"), GreaterThanEqualToAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lt"), LessThanAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("lte"), LessThanOrEqualToAssertion::parse), + new NamedXContentRegistry.Entry(ExecutableSection.class, new ParseField("length"), LengthAssertion::parse))); + + static ExecutableSection parse(XContentParser parser) throws IOException { + ParserUtils.advanceToFieldName(parser); + String section = parser.currentName(); + XContentLocation location = parser.getTokenLocation(); + try { + ExecutableSection executableSection = parser.namedObject(ExecutableSection.class, section, null); + parser.nextToken(); + return executableSection; + } catch (Exception e) { + throw new IOException("Error parsing section starting at [" + location + "]", e); + } + } + /** * Get the location in the test that this was defined. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java index b531f180fd6e3..39a1f1d378067 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanAssertion.java @@ -19,8 +19,12 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.instanceOf; @@ -33,6 +37,15 @@ * - gt: { fields._ttl: 0} */ public class GreaterThanAssertion extends Assertion { + public static GreaterThanAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + if (! (stringObjectTuple.v2() instanceof Comparable) ) { + throw new IllegalArgumentException("gt section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); + } + return new GreaterThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); + } private static final Logger logger = Loggers.getLogger(GreaterThanAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java index 14b1a08a87954..3fd9bf7adfd61 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/GreaterThanEqualToAssertion.java @@ -20,8 +20,12 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; @@ -34,6 +38,15 @@ * - gte: { fields._ttl: 0 } */ public class GreaterThanEqualToAssertion extends Assertion { + public static GreaterThanEqualToAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + if (! (stringObjectTuple.v2() instanceof Comparable) ) { + throw new IllegalArgumentException("gte section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); + } + return new GreaterThanEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); + } private static final Logger logger = Loggers.getLogger(GreaterThanEqualToAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java index a356182ab46ac..56ee603c70f75 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsFalseAssertion.java @@ -21,6 +21,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; @@ -34,6 +37,9 @@ * */ public class IsFalseAssertion extends Assertion { + public static IsFalseAssertion parse(XContentParser parser) throws IOException { + return new IsFalseAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser)); + } private static final Logger logger = Loggers.getLogger(IsFalseAssertion.class); @@ -42,7 +48,6 @@ public IsFalseAssertion(XContentLocation location, String field) { } @Override - @SuppressWarnings("unchecked") protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] doesn't have a true value (field: [{}])", actualValue, getField()); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java index 76ca0de70d996..9b3f37e1f52a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/IsTrueAssertion.java @@ -21,6 +21,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalToIgnoringCase; @@ -35,6 +38,9 @@ * */ public class IsTrueAssertion extends Assertion { + public static IsTrueAssertion parse(XContentParser parser) throws IOException { + return new IsTrueAssertion(parser.getTokenLocation(), ParserUtils.parseField(parser)); + } private static final Logger logger = Loggers.getLogger(IsTrueAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java index 062b9ecd87f34..aeecc50b90d41 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LengthAssertion.java @@ -19,9 +19,12 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -35,6 +38,22 @@ * - length: { hits.hits: 1 } */ public class LengthAssertion extends Assertion { + public static LengthAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + assert stringObjectTuple.v2() != null; + int value; + if (stringObjectTuple.v2() instanceof Number) { + value = ((Number) stringObjectTuple.v2()).intValue(); + } else { + try { + value = Integer.valueOf(stringObjectTuple.v2().toString()); + } catch(NumberFormatException e) { + throw new IllegalArgumentException("length is not a valid number", e); + } + } + return new LengthAssertion(location, stringObjectTuple.v1(), value); + } private static final Logger logger = Loggers.getLogger(LengthAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java index 591bd83fa624e..75a1edcf81caa 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanAssertion.java @@ -19,8 +19,12 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThan; @@ -34,6 +38,15 @@ * */ public class LessThanAssertion extends Assertion { + public static LessThanAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + if (false == stringObjectTuple.v2() instanceof Comparable) { + throw new IllegalArgumentException("lt section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); + } + return new LessThanAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); + } private static final Logger logger = Loggers.getLogger(LessThanAssertion.class); @@ -42,7 +55,6 @@ public LessThanAssertion(XContentLocation location, String field, Object expecte } @Override - @SuppressWarnings("unchecked") protected void doAssert(Object actualValue, Object expectedValue) { logger.trace("assert that [{}] is less than [{}] (field: [{}])", actualValue, expectedValue, getField()); assertThat("value of [" + getField() + "] is not comparable (got [" + safeClass(actualValue) + "])", diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java index 7c5710f689da3..23b6a1e4efc41 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/LessThanOrEqualToAssertion.java @@ -20,8 +20,12 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -34,6 +38,15 @@ * - lte: { fields._ttl: 0 } */ public class LessThanOrEqualToAssertion extends Assertion { + public static LessThanOrEqualToAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + if (false == stringObjectTuple.v2() instanceof Comparable) { + throw new IllegalArgumentException("lte section can only be used with objects that support natural ordering, found " + + stringObjectTuple.v2().getClass().getSimpleName()); + } + return new LessThanOrEqualToAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); + } private static final Logger logger = Loggers.getLogger(LessThanOrEqualToAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java index 60e3424704474..fb52e8ee695cb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/MatchAssertion.java @@ -20,9 +20,12 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.Map; @@ -43,6 +46,11 @@ * */ public class MatchAssertion extends Assertion { + public static MatchAssertion parse(XContentParser parser) throws IOException { + XContentLocation location = parser.getTokenLocation(); + Tuple stringObjectTuple = ParserUtils.parseTuple(parser); + return new MatchAssertion(location, stringObjectTuple.v1(), stringObjectTuple.v2()); + } private static final Logger logger = Loggers.getLogger(MatchAssertion.class); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java new file mode 100644 index 0000000000000..b35450c66e61a --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ParserUtils.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest.yaml.section; + +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Map; + +/** + * Utility methods used for parsing test sections. + */ +class ParserUtils { + private ParserUtils() { + // Do not build. + } + + public static String parseField(XContentParser parser) throws IOException { + parser.nextToken(); + assert parser.currentToken().isValue(); + String field = parser.text(); + parser.nextToken(); + return field; + } + + public static Tuple parseTuple(XContentParser parser) throws IOException { + parser.nextToken(); + advanceToFieldName(parser); + Map map = parser.map(); + assert parser.currentToken() == XContentParser.Token.END_OBJECT; + parser.nextToken(); + + if (map.size() != 1) { + throw new IllegalArgumentException("expected key value pair but found an object with " + map.size() + " fields"); + } + + Map.Entry entry = map.entrySet().iterator().next(); + return Tuple.tuple(entry.getKey(), entry.getValue()); + } + + public static void advanceToFieldName(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + //we are in the beginning, haven't called nextToken yet + if (token == null) { + token = parser.nextToken(); + } + if (token == XContentParser.Token.START_ARRAY) { + token = parser.nextToken(); + } + if (token == XContentParser.Token.START_OBJECT) { + token = parser.nextToken(); + } + if (token != XContentParser.Token.FIELD_NAME) { + throw new IllegalArgumentException("malformed test section: field name expected but found " + token + " at " + + parser.getTokenLocation()); + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java index aa39c4d30c8cb..087f667ac9bc3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetSection.java @@ -18,7 +18,9 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; import java.io.IOException; @@ -32,6 +34,28 @@ * */ public class SetSection implements ExecutableSection { + public static SetSection parse(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + + SetSection setSection = new SetSection(parser.getTokenLocation()); + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + setSection.addSet(currentFieldName, parser.text()); + } + } + + parser.nextToken(); + + if (setSection.getStash().isEmpty()) { + throw new ParsingException(setSection.location, "set section must set at least a value"); + } + + return setSection; + } private final Map stash = new HashMap<>(); private final XContentLocation location; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java index c2bcaa3ecd134..f0783cc1ccf47 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SetupSection.java @@ -18,6 +18,9 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -25,6 +28,41 @@ * Represents a setup section. Holds a skip section and multiple do sections. */ public class SetupSection { + /** + * Parse a {@link SetupSection} if the next field is {@code skip}, otherwise returns {@link SetupSection#EMPTY}. + */ + public static SetupSection parseIfNext(XContentParser parser) throws IOException { + ParserUtils.advanceToFieldName(parser); + + if ("setup".equals(parser.currentName())) { + parser.nextToken(); + SetupSection section = parse(parser); + parser.nextToken(); + return section; + } + + return EMPTY; + } + + public static SetupSection parse(XContentParser parser) throws IOException { + SetupSection setupSection = new SetupSection(); + setupSection.setSkipSection(SkipSection.parseIfNext(parser)); + + while (parser.currentToken() != XContentParser.Token.END_ARRAY) { + ParserUtils.advanceToFieldName(parser); + if (!"do".equals(parser.currentName())) { + throw new IllegalArgumentException("section [" + parser.currentName() + "] not supported within setup section"); + } + + parser.nextToken(); + setupSection.addDoSection(DoSection.parse(parser)); + parser.nextToken(); + } + + parser.nextToken(); + + return setupSection; + } public static final SetupSection EMPTY; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java index e508c3c10152a..62588baf46daf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java @@ -19,9 +19,13 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.rest.yaml.Features; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -32,6 +36,63 @@ * - a specific test feature required that might not be implemented yet by the runner */ public class SkipSection { + /** + * Parse a {@link SkipSection} if the next field is {@code skip}, otherwise returns {@link SkipSection#EMPTY}. + */ + public static SkipSection parseIfNext(XContentParser parser) throws IOException { + ParserUtils.advanceToFieldName(parser); + + if ("skip".equals(parser.currentName())) { + SkipSection section = parse(parser); + parser.nextToken(); + return section; + } + + return EMPTY; + } + + public static SkipSection parse(XContentParser parser) throws IOException { + String currentFieldName = null; + XContentParser.Token token; + String version = null; + String reason = null; + List features = new ArrayList<>(); + + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if ("version".equals(currentFieldName)) { + version = parser.text(); + } else if ("reason".equals(currentFieldName)) { + reason = parser.text(); + } else if ("features".equals(currentFieldName)) { + features.add(parser.text()); + } + else { + throw new ParsingException(parser.getTokenLocation(), + "field " + currentFieldName + " not supported within skip section"); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("features".equals(currentFieldName)) { + while(parser.nextToken() != XContentParser.Token.END_ARRAY) { + features.add(parser.text()); + } + } + } + } + + parser.nextToken(); + + if (!Strings.hasLength(version) && features.isEmpty()) { + throw new ParsingException(parser.getTokenLocation(), "version or features is mandatory within skip section"); + } + if (Strings.hasLength(version) && !Strings.hasLength(reason)) { + throw new ParsingException(parser.getTokenLocation(), "reason is mandatory within skip version section"); + } + + return new SkipSection(version, features, reason); + } public static final SkipSection EMPTY = new SkipSection(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java index 1e49f8b5037a7..1ae4d41a039b2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/TeardownSection.java @@ -19,10 +19,49 @@ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; import java.util.ArrayList; import java.util.List; public class TeardownSection { + /** + * Parse a {@link TeardownSection} if the next field is {@code skip}, otherwise returns {@link TeardownSection#EMPTY}. + */ + public static TeardownSection parseIfNext(XContentParser parser) throws IOException { + ParserUtils.advanceToFieldName(parser); + + if ("teardown".equals(parser.currentName())) { + parser.nextToken(); + TeardownSection section = parse(parser); + parser.nextToken(); + return section; + } + + return EMPTY; + } + + public static TeardownSection parse(XContentParser parser) throws IOException { + TeardownSection teardownSection = new TeardownSection(); + teardownSection.setSkipSection(SkipSection.parseIfNext(parser)); + + while (parser.currentToken() != XContentParser.Token.END_ARRAY) { + ParserUtils.advanceToFieldName(parser); + if (!"do".equals(parser.currentName())) { + throw new ParsingException(parser.getTokenLocation(), + "section [" + parser.currentName() + "] not supported within teardown section"); + } + + parser.nextToken(); + teardownSection.addDoSection(DoSection.parse(parser)); + parser.nextToken(); + } + + parser.nextToken(); + return teardownSection; + } public static final TeardownSection EMPTY; diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java deleted file mode 100644 index 6057de6c702e6..0000000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/DoSectionParserTests.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.common.xcontent.XContent; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.section.ApiCallSection; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.hamcrest.MatcherAssert; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -import static java.util.Collections.singletonList; -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; - -public class DoSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { - public void testParseDoSectionNoBody() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "get:\n" + - " index: test_index\n" + - " type: test_type\n" + - " id: 1" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("get")); - assertThat(apiCallSection.getParams().size(), equalTo(3)); - assertThat(apiCallSection.getParams().get("index"), equalTo("test_index")); - assertThat(apiCallSection.getParams().get("type"), equalTo("test_type")); - assertThat(apiCallSection.getParams().get("id"), equalTo("1")); - assertThat(apiCallSection.hasBody(), equalTo(false)); - } - - public void testParseDoSectionNoParamsNoBody() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "cluster.node_info: {}" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("cluster.node_info")); - assertThat(apiCallSection.getParams().size(), equalTo(0)); - assertThat(apiCallSection.hasBody(), equalTo(false)); - } - - public void testParseDoSectionWithJsonBody() throws Exception { - String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }"; - parser = createParser(YamlXContent.yamlXContent, - "index:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 1\n" + - " body: " + body - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("index")); - assertThat(apiCallSection.getParams().size(), equalTo(3)); - assertThat(apiCallSection.getParams().get("index"), equalTo("test_1")); - assertThat(apiCallSection.getParams().get("type"), equalTo("test")); - assertThat(apiCallSection.getParams().get("id"), equalTo("1")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - - assertJsonEquals(apiCallSection.getBodies().get(0), body); - } - - public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception { - String bodies[] = new String[]{ - "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n", - "{ \"f1\":\"v1\", \"f2\":42 }\n", - "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n", - "{ \"f1\":\"v2\", \"f2\":47 }\n" - }; - parser = createParser(YamlXContent.yamlXContent, - "bulk:\n" + - " refresh: true\n" + - " body: |\n" + - " " + bodies[0] + - " " + bodies[1] + - " " + bodies[2] + - " " + bodies[3] - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("bulk")); - assertThat(apiCallSection.getParams().size(), equalTo(1)); - assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(4)); - } - - public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception { - assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", - XContent.isStrictDuplicateDetectionEnabled()); - - String[] bodies = new String[] { - "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }", - "{ \"f1\":\"v1\", \"f2\":42 }", - }; - parser = createParser(YamlXContent.yamlXContent, - "bulk:\n" + - " refresh: true\n" + - " body: \n" + - " " + bodies[0] + "\n" + - " body: \n" + - " " + bodies[1] - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("bulk")); - assertThat(apiCallSection.getParams().size(), equalTo(1)); - assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); - for (int i = 0; i < bodies.length; i++) { - assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); - } - } - - public void testParseDoSectionWithYamlBody() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "search:\n" + - " body:\n" + - " \"_source\": [ include.field1, include.field2 ]\n" + - " \"query\": { \"match_all\": {} }" - ); - String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}"; - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("search")); - assertThat(apiCallSection.getParams().size(), equalTo(0)); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(1)); - assertJsonEquals(apiCallSection.getBodies().get(0), body); - } - - public void testParseDoSectionWithYamlMultipleBodies() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "bulk:\n" + - " refresh: true\n" + - " body:\n" + - " - index:\n" + - " _index: test_index\n" + - " _type: test_type\n" + - " _id: test_id\n" + - " - f1: v1\n" + - " f2: 42\n" + - " - index:\n" + - " _index: test_index2\n" + - " _type: test_type2\n" + - " _id: test_id2\n" + - " - f1: v2\n" + - " f2: 47" - ); - String[] bodies = new String[4]; - bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\": \"test_type\", \"_id\": \"test_id\"}}"; - bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }"; - bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\": \"test_type2\", \"_id\": \"test_id2\"}}"; - bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }"; - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("bulk")); - assertThat(apiCallSection.getParams().size(), equalTo(1)); - assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); - - for (int i = 0; i < bodies.length; i++) { - assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); - } - } - - public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception { - assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", - XContent.isStrictDuplicateDetectionEnabled()); - - parser = createParser(YamlXContent.yamlXContent, - "bulk:\n" + - " refresh: true\n" + - " body:\n" + - " index:\n" + - " _index: test_index\n" + - " _type: test_type\n" + - " _id: test_id\n" + - " body:\n" + - " f1: v1\n" + - " f2: 42\n" - ); - String[] bodies = new String[2]; - bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\": \"test_type\", \"_id\": \"test_id\"}}"; - bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }"; - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("bulk")); - assertThat(apiCallSection.getParams().size(), equalTo(1)); - assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); - - for (int i = 0; i < bodies.length; i++) { - assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); - } - } - - public void testParseDoSectionWithYamlBodyMultiGet() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "mget:\n" + - " body:\n" + - " docs:\n" + - " - { _index: test_2, _type: test, _id: 1}\n" + - " - { _index: test_1, _type: none, _id: 1}" - ); - String body = "{ \"docs\": [ " + - "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " + - "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " + - "]}"; - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("mget")); - assertThat(apiCallSection.getParams().size(), equalTo(0)); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(1)); - assertJsonEquals(apiCallSection.getBodies().get(0), body); - } - - public void testParseDoSectionWithBodyStringified() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "index:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 1\n" + - " body: \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection, notNullValue()); - assertThat(apiCallSection.getApi(), equalTo("index")); - assertThat(apiCallSection.getParams().size(), equalTo(3)); - assertThat(apiCallSection.getParams().get("index"), equalTo("test_1")); - assertThat(apiCallSection.getParams().get("type"), equalTo("test")); - assertThat(apiCallSection.getParams().get("id"), equalTo("1")); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(1)); - //stringified body is taken as is - assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }"); - } - - public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "index:\n" + - " body:\n" + - " - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" + - " - { size: 100, query: { match_all: {} } }" - ); - - String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }"; - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - ApiCallSection apiCallSection = doSection.getApiCallSection(); - - assertThat(apiCallSection.getApi(), equalTo("index")); - assertThat(apiCallSection.getParams().size(), equalTo(0)); - assertThat(apiCallSection.hasBody(), equalTo(true)); - assertThat(apiCallSection.getBodies().size(), equalTo(2)); - //stringified body is taken as is - assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }"); - assertJsonEquals(apiCallSection.getBodies().get(1), body); - } - - public void testParseDoSectionWithCatch() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "catch: missing\n" + - "indices.get_warmer:\n" + - " index: test_index\n" + - " name: test_warmer" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(doSection.getCatch(), equalTo("missing")); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - } - - public void testParseDoSectionWithHeaders() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "headers:\n" + - " Authorization: \"thing one\"\n" + - " Content-Type: \"application/json\"\n" + - "indices.get_warmer:\n" + - " index: test_index\n" + - " name: test_warmer" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - assertThat(doSection.getApiCallSection().getHeaders(), notNullValue()); - assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one")); - assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json")); - } - - public void testParseDoSectionWithoutClientCallSection() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "catch: missing\n" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - try { - doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - fail("Expected RestTestParseException"); - } catch (ClientYamlTestParseException e) { - assertThat(e.getMessage(), is("client call section is mandatory within a do section")); - } - } - - public void testParseDoSectionMultivaluedField() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "indices.get_field_mapping:\n" + - " index: test_index\n" + - " type: test_type\n" + - " field: [ text , text1 ]" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); - assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index")); - assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type")); - assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1")); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0)); - } - - public void testParseDoSectionExpectedWarnings() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "indices.get_field_mapping:\n" + - " index: test_index\n" + - " type: test_type\n" + - "warnings:\n" + - " - some test warning they are typically pretty long\n" + - " - some other test warning somtimes they have [in] them" - ); - - DoSectionParser doSectionParser = new DoSectionParser(); - DoSection doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index")); - assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type")); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0)); - assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList( - "some test warning they are typically pretty long", - "some other test warning somtimes they have [in] them"))); - - parser = createParser(YamlXContent.yamlXContent, - "indices.get_field_mapping:\n" + - " index: test_index\n" + - "warnings:\n" + - " - just one entry this time" - ); - - doSection = doSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList( - "just one entry this time"))); - - } - - private void assertJsonEquals(Map actual, String expected) throws IOException { - Map expectedMap; - try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) { - expectedMap = parser.mapOrdered(); - } - MatcherAssert.assertThat(actual, equalTo(expectedMap)); - } -} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java deleted file mode 100644 index 9f488fd9d5c7c..0000000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SkipSectionParserTests.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.Version; -import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.test.rest.yaml.section.SkipSection; - -import java.util.Arrays; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; - -public class SkipSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { - public void testParseSkipSectionVersionNoFeature() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "version: \" - 2.1.0\"\n" + - "reason: Delete ignores the parent param" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - - SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(skipSection, notNullValue()); - assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion())); - assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0)); - assertThat(skipSection.getFeatures().size(), equalTo(0)); - assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param")); - } - - public void testParseSkipSectionAllVersions() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "version: \" all \"\n" + - "reason: Delete ignores the parent param" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - - SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(skipSection, notNullValue()); - assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion())); - assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT)); - assertThat(skipSection.getFeatures().size(), equalTo(0)); - assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param")); - } - - public void testParseSkipSectionFeatureNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "features: regex" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - - SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(skipSection, notNullValue()); - assertThat(skipSection.isVersionCheck(), equalTo(false)); - assertThat(skipSection.getFeatures().size(), equalTo(1)); - assertThat(skipSection.getFeatures().get(0), equalTo("regex")); - assertThat(skipSection.getReason(), nullValue()); - } - - public void testParseSkipSectionFeaturesNoVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "features: [regex1,regex2,regex3]" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - - SkipSection skipSection = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(skipSection, notNullValue()); - assertThat(skipSection.isVersionCheck(), equalTo(false)); - assertThat(skipSection.getFeatures().size(), equalTo(3)); - assertThat(skipSection.getFeatures().get(0), equalTo("regex1")); - assertThat(skipSection.getFeatures().get(1), equalTo("regex2")); - assertThat(skipSection.getFeatures().get(2), equalTo("regex3")); - assertThat(skipSection.getReason(), nullValue()); - } - - public void testParseSkipSectionBothFeatureAndVersion() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "version: \" - 0.90.2\"\n" + - "features: regex\n" + - "reason: Delete ignores the parent param" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - SkipSection parse = skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - assertEquals(VersionUtils.getFirstVersion(), parse.getLowerVersion()); - assertEquals(Version.fromString("0.90.2"), parse.getUpperVersion()); - assertEquals(Arrays.asList("regex"), parse.getFeatures()); - assertEquals("Delete ignores the parent param", parse.getReason()); - } - - public void testParseSkipSectionNoReason() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "version: \" - 0.90.2\"\n" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - try { - skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - fail("Expected RestTestParseException"); - } catch (ClientYamlTestParseException e) { - assertThat(e.getMessage(), is("reason is mandatory within skip version section")); - } - } - - public void testParseSkipSectionNoVersionNorFeature() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "reason: Delete ignores the parent param\n" - ); - - SkipSectionParser skipSectionParser = new SkipSectionParser(); - try { - skipSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - fail("Expected RestTestParseException"); - } catch (ClientYamlTestParseException e) { - assertThat(e.getMessage(), is("version or features is mandatory within skip section")); - } - } -} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java deleted file mode 100644 index 5673f20f06569..0000000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TestSectionParserTests.java +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.test.rest.yaml.parser; - -import org.elasticsearch.Version; -import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion; -import org.elasticsearch.test.rest.yaml.section.IsFalseAssertion; -import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion; -import org.elasticsearch.test.rest.yaml.section.LengthAssertion; -import org.elasticsearch.test.rest.yaml.section.LessThanAssertion; -import org.elasticsearch.test.rest.yaml.section.MatchAssertion; -import org.elasticsearch.test.rest.yaml.section.SetSection; -import org.elasticsearch.test.rest.yaml.section.SkipSection; - -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; - -public class TestSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { - public void testParseTestSectionWithDoSection() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "\"First test section\": \n" + - " - do :\n" + - " catch: missing\n" + - " indices.get_warmer:\n" + - " index: test_index\n" + - " name: test_warmer" - ); - - ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser(); - ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(testSection, notNullValue()); - assertThat(testSection.getName(), equalTo("First test section")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); - assertThat(testSection.getExecutableSections().size(), equalTo(1)); - DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); - assertThat(doSection.getCatch(), equalTo("missing")); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - } - - public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception { - String yaml = - "\"First test section\": \n" + - " - skip:\n" + - " version: \"2.0.0 - 2.2.0\"\n" + - " reason: \"Update doesn't return metadata fields, waiting for #3259\"\n" + - " - do :\n" + - " catch: missing\n" + - " indices.get_warmer:\n" + - " index: test_index\n" + - " name: test_warmer\n" + - " - set: {_scroll_id: scroll_id}"; - - - ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser(); - parser = createParser(YamlXContent.yamlXContent,yaml); - ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(testSection, notNullValue()); - assertThat(testSection.getName(), equalTo("First test section")); - assertThat(testSection.getSkipSection(), notNullValue()); - assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0)); - assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0)); - assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); - assertThat(testSection.getExecutableSections().size(), equalTo(2)); - DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); - assertThat(doSection.getCatch(), equalTo("missing")); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - SetSection setSection = (SetSection) testSection.getExecutableSections().get(1); - assertThat(setSection.getStash().size(), equalTo(1)); - assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id")); - } - - public void testParseTestSectionWithMultipleDoSections() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "\"Basic\":\n" + - "\n" + - " - do:\n" + - " index:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 中文\n" + - " body: { \"foo\": \"Hello: 中文\" }\n" + - " - do:\n" + - " get:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 中文" - ); - - ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser(); - ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(testSection, notNullValue()); - assertThat(testSection.getName(), equalTo("Basic")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); - assertThat(testSection.getExecutableSections().size(), equalTo(2)); - DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("index")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); - doSection = (DoSection)testSection.getExecutableSections().get(1); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - } - - public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "\"Basic\":\n" + - "\n" + - " - do:\n" + - " index:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 中文\n" + - " body: { \"foo\": \"Hello: 中文\" }\n" + - "\n" + - " - do:\n" + - " get:\n" + - " index: test_1\n" + - " type: test\n" + - " id: 中文\n" + - "\n" + - " - match: { _index: test_1 }\n" + - " - is_true: _source\n" + - " - match: { _source: { foo: \"Hello: 中文\" } }\n" + - "\n" + - " - do:\n" + - " get:\n" + - " index: test_1\n" + - " id: 中文\n" + - "\n" + - " - length: { _index: 6 }\n" + - " - is_false: whatever\n" + - " - gt: { size: 5 }\n" + - " - lt: { size: 10 }" - ); - - ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser(); - ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - - assertThat(testSection, notNullValue()); - assertThat(testSection.getName(), equalTo("Basic")); - assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); - assertThat(testSection.getExecutableSections().size(), equalTo(10)); - - DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("index")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); - - doSection = (DoSection)testSection.getExecutableSections().get(1); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - - MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2); - assertThat(matchAssertion.getField(), equalTo("_index")); - assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1")); - - IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3); - assertThat(trueAssertion.getField(), equalTo("_source")); - - matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4); - assertThat(matchAssertion.getField(), equalTo("_source")); - assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class)); - Map map = (Map) matchAssertion.getExpectedValue(); - assertThat(map.size(), equalTo(1)); - assertThat(map.get("foo").toString(), equalTo("Hello: 中文")); - - doSection = (DoSection)testSection.getExecutableSections().get(5); - assertThat(doSection.getCatch(), nullValue()); - assertThat(doSection.getApiCallSection(), notNullValue()); - assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); - assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); - assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); - - LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6); - assertThat(lengthAssertion.getField(), equalTo("_index")); - assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6)); - - IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7); - assertThat(falseAssertion.getField(), equalTo("whatever")); - - GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8); - assertThat(greaterThanAssertion.getField(), equalTo("size")); - assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5)); - - LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9); - assertThat(lessThanAssertion.getField(), equalTo("size")); - assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class)); - assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10)); - } - - public void testSmallSection() throws Exception { - parser = createParser(YamlXContent.yamlXContent, - "\"node_info test\":\n" + - " - do:\n" + - " cluster.node_info: {}\n" + - " \n" + - " - is_true: nodes\n" + - " - is_true: cluster_name\n"); - ClientYamlTestSectionParser testSectionParser = new ClientYamlTestSectionParser(); - ClientYamlTestSection testSection = testSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - assertThat(testSection, notNullValue()); - assertThat(testSection.getName(), equalTo("node_info test")); - assertThat(testSection.getExecutableSections().size(), equalTo(3)); - } -} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java index caca95b131514..6acdd935400f4 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/restspec/ClientYamlSuiteRestApiParserTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.test.rest.yaml.restspec; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.parser.AbstractClientYamlTestFragmentParserTestCase; +import org.elasticsearch.test.rest.yaml.section.AbstractClientYamlTestFragmentParserTestCase; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java similarity index 69% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java index 3b0859d2d8b2e..b7038a7d7a329 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AbstractClientYamlTestFragmentParserTestCase.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AbstractClientYamlTestFragmentParserTestCase.java @@ -17,19 +17,20 @@ * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; +package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.yaml.section.ExecutableSection; import org.junit.After; import static org.hamcrest.Matchers.nullValue; /** - * Superclass for tests of subclasses of {@link ClientYamlTestFragmentParser}. + * Superclass for tests that parse parts of the test suite. */ public abstract class AbstractClientYamlTestFragmentParserTestCase extends ESTestCase { - protected XContentParser parser; @Override @@ -38,9 +39,16 @@ public void tearDown() throws Exception { super.tearDown(); // test may be skipped so we did not create a parser instance if (parser != null) { - //this is the way to make sure that we consumed the whole yaml + //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times assertThat(parser.currentToken(), nullValue()); + assertThat(parser.nextToken(), nullValue()); + assertThat(parser.nextToken(), nullValue()); parser.close(); } } + + @Override + protected NamedXContentRegistry xContentRegistry() { + return ExecutableSection.XCONTENT_REGISTRY; + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java similarity index 75% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java index c908f0791493f..fcef74678359e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/AssertionParsersTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/AssertionTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; +package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.test.rest.yaml.section.GreaterThanAssertion; @@ -33,14 +33,13 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.notNullValue; -public class AssertionParsersTests extends AbstractClientYamlTestFragmentParserTestCase { +public class AssertionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testParseIsTrue() throws Exception { parser = createParser(YamlXContent.yamlXContent, "get.fields._timestamp" ); - IsTrueParser isTrueParser = new IsTrueParser(); - IsTrueAssertion trueAssertion = isTrueParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + IsTrueAssertion trueAssertion = IsTrueAssertion.parse(parser); assertThat(trueAssertion, notNullValue()); assertThat(trueAssertion.getField(), equalTo("get.fields._timestamp")); @@ -51,8 +50,7 @@ public void testParseIsFalse() throws Exception { "docs.1._source" ); - IsFalseParser isFalseParser = new IsFalseParser(); - IsFalseAssertion falseAssertion = isFalseParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + IsFalseAssertion falseAssertion = IsFalseAssertion.parse(parser); assertThat(falseAssertion, notNullValue()); assertThat(falseAssertion.getField(), equalTo("docs.1._source")); @@ -63,8 +61,7 @@ public void testParseGreaterThan() throws Exception { "{ field: 3}" ); - GreaterThanParser greaterThanParser = new GreaterThanParser(); - GreaterThanAssertion greaterThanAssertion = greaterThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + GreaterThanAssertion greaterThanAssertion = GreaterThanAssertion.parse(parser); assertThat(greaterThanAssertion, notNullValue()); assertThat(greaterThanAssertion.getField(), equalTo("field")); assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class)); @@ -76,8 +73,7 @@ public void testParseLessThan() throws Exception { "{ field: 3}" ); - LessThanParser lessThanParser = new LessThanParser(); - LessThanAssertion lessThanAssertion = lessThanParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + LessThanAssertion lessThanAssertion = LessThanAssertion.parse(parser); assertThat(lessThanAssertion, notNullValue()); assertThat(lessThanAssertion.getField(), equalTo("field")); assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class)); @@ -89,8 +85,7 @@ public void testParseLength() throws Exception { "{ _id: 22}" ); - LengthParser lengthParser = new LengthParser(); - LengthAssertion lengthAssertion = lengthParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + LengthAssertion lengthAssertion = LengthAssertion.parse(parser); assertThat(lengthAssertion, notNullValue()); assertThat(lengthAssertion.getField(), equalTo("_id")); assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class)); @@ -102,8 +97,7 @@ public void testParseMatchSimpleIntegerValue() throws Exception { "{ field: 10 }" ); - MatchParser matchParser = new MatchParser(); - MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + MatchAssertion matchAssertion = MatchAssertion.parse(parser); assertThat(matchAssertion, notNullValue()); assertThat(matchAssertion.getField(), equalTo("field")); @@ -116,8 +110,7 @@ public void testParseMatchSimpleStringValue() throws Exception { "{ foo: bar }" ); - MatchParser matchParser = new MatchParser(); - MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + MatchAssertion matchAssertion = MatchAssertion.parse(parser); assertThat(matchAssertion, notNullValue()); assertThat(matchAssertion.getField(), equalTo("foo")); @@ -130,13 +123,12 @@ public void testParseMatchArray() throws Exception { "{'matches': ['test_percolator_1', 'test_percolator_2']}" ); - MatchParser matchParser = new MatchParser(); - MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + MatchAssertion matchAssertion = MatchAssertion.parse(parser); assertThat(matchAssertion, notNullValue()); assertThat(matchAssertion.getField(), equalTo("matches")); assertThat(matchAssertion.getExpectedValue(), instanceOf(List.class)); - List strings = (List) matchAssertion.getExpectedValue(); + List strings = (List) matchAssertion.getExpectedValue(); assertThat(strings.size(), equalTo(2)); assertThat(strings.get(0).toString(), equalTo("test_percolator_1")); assertThat(strings.get(1).toString(), equalTo("test_percolator_2")); @@ -148,8 +140,7 @@ public void testParseMatchSourceValues() throws Exception { "{ _source: { responses.0.hits.total: 3, foo: bar }}" ); - MatchParser matchParser = new MatchParser(); - MatchAssertion matchAssertion = matchParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + MatchAssertion matchAssertion = MatchAssertion.parse(parser); assertThat(matchAssertion, notNullValue()); assertThat(matchAssertion.getField(), equalTo("_source")); diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index db12c3af9b9e4..0947c834513c2 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -19,15 +19,22 @@ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; + +import java.util.Map; import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; -public class ClientYamlTestSectionTests extends ESTestCase { +public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testAddingDoWithoutWarningWithoutSkip() { int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection("test"); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); section.setSkipSection(SkipSection.EMPTY); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); section.addExecutableSection(doSection); @@ -35,7 +42,7 @@ public void testAddingDoWithoutWarningWithoutSkip() { public void testAddingDoWithWarningWithSkip() { int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection("test"); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); section.setSkipSection(new SkipSection(null, singletonList("warnings"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); @@ -44,7 +51,7 @@ public void testAddingDoWithWarningWithSkip() { public void testAddingDoWithWarningWithSkipButNotWarnings() { int lineNumber = between(1, 10000); - ClientYamlTestSection section = new ClientYamlTestSection("test"); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); @@ -53,4 +60,207 @@ public void testAddingDoWithWarningWithSkipButNotWarnings() { + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage()); } + public void testParseTestSectionWithDoSection() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "\"First test section\": \n" + + " - do :\n" + + " catch: missing\n" + + " indices.get_warmer:\n" + + " index: test_index\n" + + " name: test_warmer" + ); + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + + assertThat(testSection, notNullValue()); + assertThat(testSection.getName(), equalTo("First test section")); + assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getExecutableSections().size(), equalTo(1)); + DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); + assertThat(doSection.getCatch(), equalTo("missing")); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + } + + public void testParseTestSectionWithDoSetAndSkipSectionsNoSkip() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "\"First test section\": \n" + + " - skip:\n" + + " version: \"2.0.0 - 2.2.0\"\n" + + " reason: \"Update doesn't return metadata fields, waiting for #3259\"\n" + + " - do :\n" + + " catch: missing\n" + + " indices.get_warmer:\n" + + " index: test_index\n" + + " name: test_warmer\n" + + " - set: {_scroll_id: scroll_id}"); + + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + + assertThat(testSection, notNullValue()); + assertThat(testSection.getName(), equalTo("First test section")); + assertThat(testSection.getSkipSection(), notNullValue()); + assertThat(testSection.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0)); + assertThat(testSection.getSkipSection().getUpperVersion(), equalTo(Version.V_2_2_0)); + assertThat(testSection.getSkipSection().getReason(), equalTo("Update doesn't return metadata fields, waiting for #3259")); + assertThat(testSection.getExecutableSections().size(), equalTo(2)); + DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); + assertThat(doSection.getCatch(), equalTo("missing")); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + SetSection setSection = (SetSection) testSection.getExecutableSections().get(1); + assertThat(setSection.getStash().size(), equalTo(1)); + assertThat(setSection.getStash().get("_scroll_id"), equalTo("scroll_id")); + } + + public void testParseTestSectionWithMultipleDoSections() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "\"Basic\":\n" + + "\n" + + " - do:\n" + + " index:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 中文\n" + + " body: { \"foo\": \"Hello: 中文\" }\n" + + " - do:\n" + + " get:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 中文" + ); + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + + assertThat(testSection, notNullValue()); + assertThat(testSection.getName(), equalTo("Basic")); + assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getExecutableSections().size(), equalTo(2)); + DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("index")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); + doSection = (DoSection)testSection.getExecutableSections().get(1); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + } + + public void testParseTestSectionWithDoSectionsAndAssertions() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "\"Basic\":\n" + + "\n" + + " - do:\n" + + " index:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 中文\n" + + " body: { \"foo\": \"Hello: 中文\" }\n" + + "\n" + + " - do:\n" + + " get:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 中文\n" + + "\n" + + " - match: { _index: test_1 }\n" + + " - is_true: _source\n" + + " - match: { _source: { foo: \"Hello: 中文\" } }\n" + + "\n" + + " - do:\n" + + " get:\n" + + " index: test_1\n" + + " id: 中文\n" + + "\n" + + " - length: { _index: 6 }\n" + + " - is_false: whatever\n" + + " - gt: { size: 5 }\n" + + " - lt: { size: 10 }" + ); + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + + assertThat(testSection, notNullValue()); + assertThat(testSection.getName(), equalTo("Basic")); + assertThat(testSection.getSkipSection(), equalTo(SkipSection.EMPTY)); + assertThat(testSection.getExecutableSections().size(), equalTo(10)); + + DoSection doSection = (DoSection)testSection.getExecutableSections().get(0); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("index")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(true)); + + doSection = (DoSection)testSection.getExecutableSections().get(1); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + + MatchAssertion matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(2); + assertThat(matchAssertion.getField(), equalTo("_index")); + assertThat(matchAssertion.getExpectedValue().toString(), equalTo("test_1")); + + IsTrueAssertion trueAssertion = (IsTrueAssertion)testSection.getExecutableSections().get(3); + assertThat(trueAssertion.getField(), equalTo("_source")); + + matchAssertion = (MatchAssertion)testSection.getExecutableSections().get(4); + assertThat(matchAssertion.getField(), equalTo("_source")); + assertThat(matchAssertion.getExpectedValue(), instanceOf(Map.class)); + Map map = (Map) matchAssertion.getExpectedValue(); + assertThat(map.size(), equalTo(1)); + assertThat(map.get("foo").toString(), equalTo("Hello: 中文")); + + doSection = (DoSection)testSection.getExecutableSections().get(5); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("get")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + + LengthAssertion lengthAssertion = (LengthAssertion) testSection.getExecutableSections().get(6); + assertThat(lengthAssertion.getField(), equalTo("_index")); + assertThat(lengthAssertion.getExpectedValue(), instanceOf(Integer.class)); + assertThat((Integer) lengthAssertion.getExpectedValue(), equalTo(6)); + + IsFalseAssertion falseAssertion = (IsFalseAssertion)testSection.getExecutableSections().get(7); + assertThat(falseAssertion.getField(), equalTo("whatever")); + + GreaterThanAssertion greaterThanAssertion = (GreaterThanAssertion) testSection.getExecutableSections().get(8); + assertThat(greaterThanAssertion.getField(), equalTo("size")); + assertThat(greaterThanAssertion.getExpectedValue(), instanceOf(Integer.class)); + assertThat((Integer) greaterThanAssertion.getExpectedValue(), equalTo(5)); + + LessThanAssertion lessThanAssertion = (LessThanAssertion) testSection.getExecutableSections().get(9); + assertThat(lessThanAssertion.getField(), equalTo("size")); + assertThat(lessThanAssertion.getExpectedValue(), instanceOf(Integer.class)); + assertThat((Integer) lessThanAssertion.getExpectedValue(), equalTo(10)); + } + + public void testSmallSection() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "\"node_info test\":\n" + + " - do:\n" + + " cluster.node_info: {}\n" + + " \n" + + " - is_true: nodes\n" + + " - is_true: cluster_name\n"); + + ClientYamlTestSection testSection = ClientYamlTestSection.parse(parser); + assertThat(testSection, notNullValue()); + assertThat(testSection.getName(), equalTo("node_info test")); + assertThat(testSection.getExecutableSections().size(), equalTo(3)); + } + } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java similarity index 90% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java index 801bc4e06e40a..4c96986146b5e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/ClientYamlSuiteTestParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuiteTests.java @@ -16,20 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; + +package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; -import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParser; -import org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite; -import org.elasticsearch.test.rest.yaml.section.DoSection; -import org.elasticsearch.test.rest.yaml.section.IsTrueAssertion; -import org.elasticsearch.test.rest.yaml.section.MatchAssertion; -import org.junit.After; import java.util.Map; @@ -39,21 +31,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class ClientYamlSuiteTestParserTests extends ESTestCase { - private XContentParser parser; - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - //makes sure that we consumed the whole stream, XContentParser doesn't expose isClosed method - //next token can be null even in the middle of the document (e.g. with "---"), but not too many consecutive times - assertThat(parser.currentToken(), nullValue()); - assertThat(parser.nextToken(), nullValue()); - assertThat(parser.nextToken(), nullValue()); - parser.close(); - } - +public class ClientYamlTestSuiteTests extends AbstractClientYamlTestFragmentParserTestCase { public void testParseTestSetupTeardownAndSections() throws Exception { final boolean includeSetup = randomBoolean(); final boolean includeTeardown = randomBoolean(); @@ -103,11 +81,10 @@ public void testParseTestSetupTeardownAndSections() throws Exception { " - match: {test_type.properties.text.analyzer: whitespace}\n" ); - ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser(); - ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser); assertThat(restTestSuite, notNullValue()); - assertThat(restTestSuite.getName(), equalTo("suite")); + assertThat(restTestSuite.getName(), equalTo(getTestName())); assertThat(restTestSuite.getSetupSection(), notNullValue()); if (includeSetup) { assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(false)); @@ -207,11 +184,10 @@ public void testParseTestSingleTestSection() throws Exception { " - match: { _source: { foo: bar }}" ); - ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser(); - ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser); assertThat(restTestSuite, notNullValue()); - assertThat(restTestSuite.getName(), equalTo("suite")); + assertThat(restTestSuite.getName(), equalTo(getTestName())); assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true)); @@ -320,11 +296,10 @@ public void testParseTestMultipleTestSections() throws Exception { " params: { bar: 'xxx' }\n" ); - ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser(); - ClientYamlTestSuite restTestSuite = testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + ClientYamlTestSuite restTestSuite = ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser); assertThat(restTestSuite, notNullValue()); - assertThat(restTestSuite.getName(), equalTo("suite")); + assertThat(restTestSuite.getName(), equalTo(getTestName())); assertThat(restTestSuite.getSetupSection().isEmpty(), equalTo(true)); @@ -394,12 +369,8 @@ public void testParseTestDuplicateTestSections() throws Exception { "\n" ); - ClientYamlTestSuiteParser testParser = new ClientYamlTestSuiteParser(); - try { - testParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - fail("Expected RestTestParseException"); - } catch (ClientYamlTestParseException e) { - assertThat(e.getMessage(), containsString("duplicate test section")); - } + Exception e = expectThrows(ParsingException.class, () -> + ClientYamlTestSuite.parse(getTestClass().getName(), getTestName(), parser)); + assertThat(e.getMessage(), containsString("duplicate test section")); } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index e981b2d999c06..463e71341bb73 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -19,16 +19,24 @@ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentLocation; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import org.hamcrest.MatcherAssert; import java.io.IOException; import java.util.Arrays; +import java.util.Map; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; -public class DoSectionTests extends ESTestCase { +public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testWarningHeaders() throws IOException { DoSection section = new DoSection(new XContentLocation(1, 1)); @@ -63,4 +71,397 @@ public void testWarningHeaders() throws IOException { assertEquals("got unexpected warning headers [\ncat\n] didn't get expected warning headers [\nanother\nsome more\n]", e.getMessage()); } + + public void testParseDoSectionNoBody() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "get:\n" + + " index: test_index\n" + + " type: test_type\n" + + " id: 1" + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("get")); + assertThat(apiCallSection.getParams().size(), equalTo(3)); + assertThat(apiCallSection.getParams().get("index"), equalTo("test_index")); + assertThat(apiCallSection.getParams().get("type"), equalTo("test_type")); + assertThat(apiCallSection.getParams().get("id"), equalTo("1")); + assertThat(apiCallSection.hasBody(), equalTo(false)); + } + + public void testParseDoSectionNoParamsNoBody() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "cluster.node_info: {}" + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("cluster.node_info")); + assertThat(apiCallSection.getParams().size(), equalTo(0)); + assertThat(apiCallSection.hasBody(), equalTo(false)); + } + + public void testParseDoSectionWithJsonBody() throws Exception { + String body = "{ \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }"; + parser = createParser(YamlXContent.yamlXContent, + "index:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 1\n" + + " body: " + body + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("index")); + assertThat(apiCallSection.getParams().size(), equalTo(3)); + assertThat(apiCallSection.getParams().get("index"), equalTo("test_1")); + assertThat(apiCallSection.getParams().get("type"), equalTo("test")); + assertThat(apiCallSection.getParams().get("id"), equalTo("1")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + + assertJsonEquals(apiCallSection.getBodies().get(0), body); + } + + public void testParseDoSectionWithJsonMultipleBodiesAsLongString() throws Exception { + String bodies[] = new String[]{ + "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }\n", + "{ \"f1\":\"v1\", \"f2\":42 }\n", + "{ \"index\": { \"_index\":\"test_index2\", \"_type\":\"test_type2\", \"_id\":\"test_id2\" } }\n", + "{ \"f1\":\"v2\", \"f2\":47 }\n" + }; + parser = createParser(YamlXContent.yamlXContent, + "bulk:\n" + + " refresh: true\n" + + " body: |\n" + + " " + bodies[0] + + " " + bodies[1] + + " " + bodies[2] + + " " + bodies[3] + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("bulk")); + assertThat(apiCallSection.getParams().size(), equalTo(1)); + assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(4)); + } + + public void testParseDoSectionWithJsonMultipleBodiesRepeatedProperty() throws Exception { + assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", + XContent.isStrictDuplicateDetectionEnabled()); + + String[] bodies = new String[] { + "{ \"index\": { \"_index\":\"test_index\", \"_type\":\"test_type\", \"_id\":\"test_id\" } }", + "{ \"f1\":\"v1\", \"f2\":42 }", + }; + parser = createParser(YamlXContent.yamlXContent, + "bulk:\n" + + " refresh: true\n" + + " body: \n" + + " " + bodies[0] + "\n" + + " body: \n" + + " " + bodies[1] + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("bulk")); + assertThat(apiCallSection.getParams().size(), equalTo(1)); + assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); + for (int i = 0; i < bodies.length; i++) { + assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); + } + } + + public void testParseDoSectionWithYamlBody() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "search:\n" + + " body:\n" + + " \"_source\": [ include.field1, include.field2 ]\n" + + " \"query\": { \"match_all\": {} }" + ); + String body = "{ \"_source\": [ \"include.field1\", \"include.field2\" ], \"query\": { \"match_all\": {} }}"; + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("search")); + assertThat(apiCallSection.getParams().size(), equalTo(0)); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(1)); + assertJsonEquals(apiCallSection.getBodies().get(0), body); + } + + public void testParseDoSectionWithYamlMultipleBodies() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "bulk:\n" + + " refresh: true\n" + + " body:\n" + + " - index:\n" + + " _index: test_index\n" + + " _type: test_type\n" + + " _id: test_id\n" + + " - f1: v1\n" + + " f2: 42\n" + + " - index:\n" + + " _index: test_index2\n" + + " _type: test_type2\n" + + " _id: test_id2\n" + + " - f1: v2\n" + + " f2: 47" + ); + String[] bodies = new String[4]; + bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\": \"test_type\", \"_id\": \"test_id\"}}"; + bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }"; + bodies[2] = "{\"index\": {\"_index\": \"test_index2\", \"_type\": \"test_type2\", \"_id\": \"test_id2\"}}"; + bodies[3] = "{ \"f1\":\"v2\", \"f2\": 47 }"; + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("bulk")); + assertThat(apiCallSection.getParams().size(), equalTo(1)); + assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); + + for (int i = 0; i < bodies.length; i++) { + assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); + } + } + + public void testParseDoSectionWithYamlMultipleBodiesRepeatedProperty() throws Exception { + assumeFalse("Test only makes sense if XContent parser doesn't have strict duplicate checks enabled", + XContent.isStrictDuplicateDetectionEnabled()); + + parser = createParser(YamlXContent.yamlXContent, + "bulk:\n" + + " refresh: true\n" + + " body:\n" + + " index:\n" + + " _index: test_index\n" + + " _type: test_type\n" + + " _id: test_id\n" + + " body:\n" + + " f1: v1\n" + + " f2: 42\n" + ); + String[] bodies = new String[2]; + bodies[0] = "{\"index\": {\"_index\": \"test_index\", \"_type\": \"test_type\", \"_id\": \"test_id\"}}"; + bodies[1] = "{ \"f1\":\"v1\", \"f2\": 42 }"; + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("bulk")); + assertThat(apiCallSection.getParams().size(), equalTo(1)); + assertThat(apiCallSection.getParams().get("refresh"), equalTo("true")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(bodies.length)); + + for (int i = 0; i < bodies.length; i++) { + assertJsonEquals(apiCallSection.getBodies().get(i), bodies[i]); + } + } + + public void testParseDoSectionWithYamlBodyMultiGet() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "mget:\n" + + " body:\n" + + " docs:\n" + + " - { _index: test_2, _type: test, _id: 1}\n" + + " - { _index: test_1, _type: none, _id: 1}" + ); + String body = "{ \"docs\": [ " + + "{\"_index\": \"test_2\", \"_type\":\"test\", \"_id\":1}, " + + "{\"_index\": \"test_1\", \"_type\":\"none\", \"_id\":1} " + + "]}"; + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("mget")); + assertThat(apiCallSection.getParams().size(), equalTo(0)); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(1)); + assertJsonEquals(apiCallSection.getBodies().get(0), body); + } + + public void testParseDoSectionWithBodyStringified() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "index:\n" + + " index: test_1\n" + + " type: test\n" + + " id: 1\n" + + " body: \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"" + ); + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection, notNullValue()); + assertThat(apiCallSection.getApi(), equalTo("index")); + assertThat(apiCallSection.getParams().size(), equalTo(3)); + assertThat(apiCallSection.getParams().get("index"), equalTo("test_1")); + assertThat(apiCallSection.getParams().get("type"), equalTo("test")); + assertThat(apiCallSection.getParams().get("id"), equalTo("1")); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(1)); + //stringified body is taken as is + assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }"); + } + + public void testParseDoSectionWithBodiesStringifiedAndNot() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "index:\n" + + " body:\n" + + " - \"{ \\\"_source\\\": true, \\\"query\\\": { \\\"match_all\\\": {} } }\"\n" + + " - { size: 100, query: { match_all: {} } }" + ); + + String body = "{ \"size\": 100, \"query\": { \"match_all\": {} } }"; + + DoSection doSection = DoSection.parse(parser); + ApiCallSection apiCallSection = doSection.getApiCallSection(); + + assertThat(apiCallSection.getApi(), equalTo("index")); + assertThat(apiCallSection.getParams().size(), equalTo(0)); + assertThat(apiCallSection.hasBody(), equalTo(true)); + assertThat(apiCallSection.getBodies().size(), equalTo(2)); + //stringified body is taken as is + assertJsonEquals(apiCallSection.getBodies().get(0), "{ \"_source\": true, \"query\": { \"match_all\": {} } }"); + assertJsonEquals(apiCallSection.getBodies().get(1), body); + } + + public void testParseDoSectionWithCatch() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "catch: missing\n" + + "indices.get_warmer:\n" + + " index: test_index\n" + + " name: test_warmer" + ); + + DoSection doSection = DoSection.parse(parser); + assertThat(doSection.getCatch(), equalTo("missing")); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + } + + public void testParseDoSectionWithHeaders() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "headers:\n" + + " Authorization: \"thing one\"\n" + + " Content-Type: \"application/json\"\n" + + "indices.get_warmer:\n" + + " index: test_index\n" + + " name: test_warmer" + ); + + DoSection doSection = DoSection.parse(parser); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_warmer")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + assertThat(doSection.getApiCallSection().getHeaders(), notNullValue()); + assertThat(doSection.getApiCallSection().getHeaders().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().getHeaders().get("Authorization"), equalTo("thing one")); + assertThat(doSection.getApiCallSection().getHeaders().get("Content-Type"), equalTo("application/json")); + } + + public void testParseDoSectionWithoutClientCallSection() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "catch: missing\n" + ); + + Exception e = expectThrows(IllegalArgumentException.class, () -> DoSection.parse(parser)); + assertThat(e.getMessage(), is("client call section is mandatory within a do section")); + } + + public void testParseDoSectionMultivaluedField() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "indices.get_field_mapping:\n" + + " index: test_index\n" + + " type: test_type\n" + + " field: [ text , text1 ]" + ); + + DoSection doSection = DoSection.parse(parser); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(3)); + assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index")); + assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type")); + assertThat(doSection.getApiCallSection().getParams().get("field"), equalTo("text,text1")); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0)); + } + + public void testParseDoSectionExpectedWarnings() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "indices.get_field_mapping:\n" + + " index: test_index\n" + + " type: test_type\n" + + "warnings:\n" + + " - some test warning they are typically pretty long\n" + + " - some other test warning somtimes they have [in] them" + ); + + DoSection doSection = DoSection.parse(parser); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getApiCallSection().getApi(), equalTo("indices.get_field_mapping")); + assertThat(doSection.getApiCallSection().getParams().size(), equalTo(2)); + assertThat(doSection.getApiCallSection().getParams().get("index"), equalTo("test_index")); + assertThat(doSection.getApiCallSection().getParams().get("type"), equalTo("test_type")); + assertThat(doSection.getApiCallSection().hasBody(), equalTo(false)); + assertThat(doSection.getApiCallSection().getBodies().size(), equalTo(0)); + assertThat(doSection.getExpectedWarningHeaders(), equalTo(Arrays.asList( + "some test warning they are typically pretty long", + "some other test warning somtimes they have [in] them"))); + + parser = createParser(YamlXContent.yamlXContent, + "indices.get_field_mapping:\n" + + " index: test_index\n" + + "warnings:\n" + + " - just one entry this time" + ); + + doSection = DoSection.parse(parser); + assertThat(doSection.getCatch(), nullValue()); + assertThat(doSection.getApiCallSection(), notNullValue()); + assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList( + "just one entry this time"))); + + } + + private void assertJsonEquals(Map actual, String expected) throws IOException { + Map expectedMap; + try (XContentParser parser = createParser(YamlXContent.yamlXContent, expected)) { + expectedMap = parser.mapOrdered(); + } + MatcherAssert.assertThat(actual, equalTo(expectedMap)); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java similarity index 69% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java index 91d35758785a3..74b3d774bd96f 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetSectionParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetSectionTests.java @@ -16,25 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; +package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.section.SetSection; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SetSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { +public class SetSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testParseSetSectionSingleValue() throws Exception { parser = createParser(YamlXContent.yamlXContent, "{ _id: id }" ); - SetSectionParser setSectionParser = new SetSectionParser(); - - SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - + SetSection setSection = SetSection.parse(parser); assertThat(setSection, notNullValue()); assertThat(setSection.getStash(), notNullValue()); assertThat(setSection.getStash().size(), equalTo(1)); @@ -46,10 +43,7 @@ public void testParseSetSectionMultipleValues() throws Exception { "{ _id: id, _type: type, _index: index }" ); - SetSectionParser setSectionParser = new SetSectionParser(); - - SetSection setSection = setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - + SetSection setSection = SetSection.parse(parser); assertThat(setSection, notNullValue()); assertThat(setSection.getStash(), notNullValue()); assertThat(setSection.getStash().size(), equalTo(3)); @@ -63,12 +57,7 @@ public void testParseSetSectionNoValues() throws Exception { "{ }" ); - SetSectionParser setSectionParser = new SetSectionParser(); - try { - setSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - fail("Expected RestTestParseException"); - } catch (ClientYamlTestParseException e) { - assertThat(e.getMessage(), is("set section must set at least a value")); - } + Exception e = expectThrows(ParsingException.class, () -> SetSection.parse(parser)); + assertThat(e.getMessage(), is("set section must set at least a value")); } } \ No newline at end of file diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java similarity index 88% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java index e18e4a3e78e9b..f6174cf0be207 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/SetupSectionParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SetupSectionTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; +package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.yaml.YamlXContent; @@ -25,7 +25,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -public class SetupSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { +public class SetupSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testParseSetupSection() throws Exception { parser = createParser(YamlXContent.yamlXContent, " - do:\n" + @@ -42,8 +42,7 @@ public void testParseSetupSection() throws Exception { " body: { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" ); - SetupSectionParser setupSectionParser = new SetupSectionParser(); - SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + SetupSection setupSection = SetupSection.parse(parser); assertThat(setupSection, notNullValue()); assertThat(setupSection.getSkipSection().isEmpty(), equalTo(true)); @@ -71,8 +70,7 @@ public void testParseSetupAndSkipSectionNoSkip() throws Exception { " body: { \"include\": { \"field1\": \"v1\", \"field2\": \"v2\" }, \"count\": 1 }\n" ); - SetupSectionParser setupSectionParser = new SetupSectionParser(); - SetupSection setupSection = setupSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); + SetupSection setupSection = SetupSection.parse(parser); assertThat(setupSection, notNullValue()); assertThat(setupSection.getSkipSection().isEmpty(), equalTo(false)); diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java index c8f7b3512824d..984cbf1715cbf 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/SkipSectionTests.java @@ -20,12 +20,19 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import org.elasticsearch.test.VersionUtils; import java.util.Arrays; import java.util.Collections; -public class SkipSectionTests extends ESTestCase { +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +public class SkipSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testSkip() { SkipSection section = new SkipSection("2.0.0 - 2.1.0", randomBoolean() ? Collections.emptyList() : @@ -44,4 +51,92 @@ public void testMessage() { section = new SkipSection(null, Arrays.asList("warnings"), null); assertEquals("[FOOBAR] skipped, unsupported features [warnings]", section.getSkipMessage("FOOBAR")); } + + public void testParseSkipSectionVersionNoFeature() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "version: \" - 2.1.0\"\n" + + "reason: Delete ignores the parent param" + ); + + SkipSection skipSection = SkipSection.parse(parser); + assertThat(skipSection, notNullValue()); + assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion())); + assertThat(skipSection.getUpperVersion(), equalTo(Version.V_2_1_0)); + assertThat(skipSection.getFeatures().size(), equalTo(0)); + assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param")); + } + + public void testParseSkipSectionAllVersions() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "version: \" all \"\n" + + "reason: Delete ignores the parent param" + ); + + SkipSection skipSection = SkipSection.parse(parser); + assertThat(skipSection, notNullValue()); + assertThat(skipSection.getLowerVersion(), equalTo(VersionUtils.getFirstVersion())); + assertThat(skipSection.getUpperVersion(), equalTo(Version.CURRENT)); + assertThat(skipSection.getFeatures().size(), equalTo(0)); + assertThat(skipSection.getReason(), equalTo("Delete ignores the parent param")); + } + + public void testParseSkipSectionFeatureNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "features: regex" + ); + + SkipSection skipSection = SkipSection.parse(parser); + assertThat(skipSection, notNullValue()); + assertThat(skipSection.isVersionCheck(), equalTo(false)); + assertThat(skipSection.getFeatures().size(), equalTo(1)); + assertThat(skipSection.getFeatures().get(0), equalTo("regex")); + assertThat(skipSection.getReason(), nullValue()); + } + + public void testParseSkipSectionFeaturesNoVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "features: [regex1,regex2,regex3]" + ); + + SkipSection skipSection = SkipSection.parse(parser); + assertThat(skipSection, notNullValue()); + assertThat(skipSection.isVersionCheck(), equalTo(false)); + assertThat(skipSection.getFeatures().size(), equalTo(3)); + assertThat(skipSection.getFeatures().get(0), equalTo("regex1")); + assertThat(skipSection.getFeatures().get(1), equalTo("regex2")); + assertThat(skipSection.getFeatures().get(2), equalTo("regex3")); + assertThat(skipSection.getReason(), nullValue()); + } + + public void testParseSkipSectionBothFeatureAndVersion() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "version: \" - 0.90.2\"\n" + + "features: regex\n" + + "reason: Delete ignores the parent param" + ); + + SkipSection skipSection = SkipSection.parse(parser); + assertEquals(VersionUtils.getFirstVersion(), skipSection.getLowerVersion()); + assertEquals(Version.fromString("0.90.2"), skipSection.getUpperVersion()); + assertEquals(Arrays.asList("regex"), skipSection.getFeatures()); + assertEquals("Delete ignores the parent param", skipSection.getReason()); + } + + public void testParseSkipSectionNoReason() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "version: \" - 0.90.2\"\n" + ); + + Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser)); + assertThat(e.getMessage(), is("reason is mandatory within skip version section")); + } + + public void testParseSkipSectionNoVersionNorFeature() throws Exception { + parser = createParser(YamlXContent.yamlXContent, + "reason: Delete ignores the parent param\n" + ); + + Exception e = expectThrows(ParsingException.class, () -> SkipSection.parse(parser)); + assertThat(e.getMessage(), is("version or features is mandatory within skip section")); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java similarity index 80% rename from test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java rename to test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java index e84eb6b681880..f057d0d370d63 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/parser/TeardownSectionParserTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/TeardownSectionTests.java @@ -17,22 +17,18 @@ * under the License. */ -package org.elasticsearch.test.rest.yaml.parser; +package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.yaml.YamlXContent; -import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestSuiteParseContext; -import org.elasticsearch.test.rest.yaml.parser.TeardownSectionParser; -import org.elasticsearch.test.rest.yaml.section.TeardownSection; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; /** - * Unit tests for the teardown section parser + * Unit tests for the teardown section. */ -public class TeardownSectionParserTests extends AbstractClientYamlTestFragmentParserTestCase { - +public class TeardownSectionTests extends AbstractClientYamlTestFragmentParserTestCase { public void testParseTeardownSection() throws Exception { parser = createParser(YamlXContent.yamlXContent, " - do:\n" + @@ -49,9 +45,7 @@ public void testParseTeardownSection() throws Exception { " ignore: 404" ); - TeardownSectionParser teardownSectionParser = new TeardownSectionParser(); - TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - + TeardownSection section = TeardownSection.parse(parser); assertThat(section, notNullValue()); assertThat(section.getSkipSection().isEmpty(), equalTo(true)); assertThat(section.getDoSections().size(), equalTo(2)); @@ -78,9 +72,7 @@ public void testParseWithSkip() throws Exception { " ignore: 404" ); - TeardownSectionParser teardownSectionParser = new TeardownSectionParser(); - TeardownSection section = teardownSectionParser.parse(new ClientYamlTestSuiteParseContext("api", "suite", parser)); - + TeardownSection section = TeardownSection.parse(parser); assertThat(section, notNullValue()); assertThat(section.getSkipSection().isEmpty(), equalTo(false)); assertThat(section.getSkipSection().getLowerVersion(), equalTo(Version.V_2_0_0)); From 6d7261c4d3a110e7a3f67216d992bfed0110111e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 22 Dec 2016 08:24:00 -0800 Subject: [PATCH 18/22] Adds ingest processor headers to exception for unknown processor. (#22315) Optimistically check for `tag` of an unknown processor for better tracking of which processor declaration is to blame in an invalid configuration. Closes #21429. --- .../org/elasticsearch/ingest/ConfigurationUtils.java | 4 ++-- .../elasticsearch/ingest/ConfigurationUtilsTests.java | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index 88105420e1498..8eaf44048c130 100644 --- a/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -280,6 +280,7 @@ private static void addHeadersToException(ElasticsearchException exception, Stri public static Processor readProcessor(Map processorFactories, String type, Map config) throws Exception { + String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); Processor.Factory factory = processorFactories.get(type); if (factory != null) { boolean ignoreFailure = ConfigurationUtils.readBooleanProperty(null, null, config, "ignore_failure", false); @@ -287,7 +288,6 @@ public static Processor readProcessor(Map processorFa ConfigurationUtils.readOptionalList(null, null, config, Pipeline.ON_FAILURE_KEY); List onFailureProcessors = readProcessorConfigs(onFailureProcessorConfigs, processorFactories); - String tag = ConfigurationUtils.readOptionalStringProperty(null, null, config, TAG_KEY); if (onFailureProcessorConfigs != null && onFailureProcessors.isEmpty()) { throw newConfigurationException(type, tag, Pipeline.ON_FAILURE_KEY, @@ -309,6 +309,6 @@ public static Processor readProcessor(Map processorFa throw newConfigurationException(type, tag, null, e); } } - throw new ElasticsearchParseException("No processor type exists with name [" + type + "]"); + throw newConfigurationException(type, tag, null, "No processor type exists with name [" + type + "]"); } } diff --git a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java index f2d9aaa517060..eacc93a96fd99 100644 --- a/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java +++ b/core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java @@ -31,6 +31,8 @@ import org.junit.Before; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; @@ -105,12 +107,17 @@ public void testReadProcessors() throws Exception { assertThat(result.get(0), sameInstance(processor)); assertThat(result.get(1), sameInstance(processor)); - config.add(Collections.singletonMap("unknown_processor", emptyConfig)); + Map unknownTaggedConfig = new HashMap<>(); + unknownTaggedConfig.put("tag", "my_unknown"); + config.add(Collections.singletonMap("unknown_processor", unknownTaggedConfig)); try { ConfigurationUtils.readProcessorConfigs(config, registry); fail("exception expected"); } catch (ElasticsearchParseException e) { assertThat(e.getMessage(), equalTo("No processor type exists with name [unknown_processor]")); + assertThat(e.getHeader("processor_tag"), equalTo(Collections.singletonList("my_unknown"))); + assertThat(e.getHeader("processor_type"), equalTo(Collections.singletonList("unknown_processor"))); + assertThat(e.getHeader("property_name"), is(nullValue())); } } From 7d0dbd2082c3f9aac9ddd64f5ec75a8b22527e51 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 22 Dec 2016 18:10:15 +0100 Subject: [PATCH 19/22] add trace logging to UnicastZenPingTests.testResolveReuseExistingNodeConnections --- .../org/elasticsearch/discovery/zen/UnicastZenPingTests.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java index 64d62336dd9c0..db81956865a06 100644 --- a/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java +++ b/core/src/test/java/org/elasticsearch/discovery/zen/UnicastZenPingTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -115,6 +116,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { try { + logger.info("shutting down..."); // JDK stack is broken, it does not iterate in the expected order (http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4475301) final List reverse = new ArrayList<>(); while (!closeables.isEmpty()) { @@ -538,6 +540,7 @@ public TransportAddress[] addressesFromString(String address, int perAddressLimi } } + @TestLogging("org.elasticsearch:DEBUG,org.elasticsearch.discovery:TRACE,org.elasticsearch.transport:TRACE") public void testResolveReuseExistingNodeConnections() throws ExecutionException, InterruptedException { final Settings settings = Settings.builder().put("cluster.name", "test").put(TransportSettings.PORT.getKey(), 0).build(); From 864e3128e76608ddcb8ec0aa8a957ee5fe21b617 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 22 Dec 2016 11:24:13 -0600 Subject: [PATCH 20/22] Add task to clean idea build directory. Make cleanIdea task invoke it. --- build.gradle | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/build.gradle b/build.gradle index ba19a993bdc60..1159352cd5dec 100644 --- a/build.gradle +++ b/build.gradle @@ -204,6 +204,14 @@ allprojects { } } } + + task cleanIdeaBuildDir(type: Delete) { + delete 'build-idea' + } + cleanIdeaBuildDir.setGroup("ide") + cleanIdeaBuildDir.setDescription("Deletes the IDEA build directory.") + + tasks.cleanIdea.dependsOn(cleanIdeaBuildDir) } idea { From 13c5881f3ec697f53ec44449c66df8f39323d513 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 22 Dec 2016 18:45:44 +0100 Subject: [PATCH 21/22] UnicastZenPing's PingingRound should prevent opening connections after being closed This may cause them to leak. Provisioning for it was made in #22277 but sadly a crucial ensureOpen call was forgotten --- .../java/org/elasticsearch/discovery/zen/UnicastZenPing.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java index 8018c47b4e113..4e2ed19f4224c 100644 --- a/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java +++ b/core/src/main/java/org/elasticsearch/discovery/zen/UnicastZenPing.java @@ -385,12 +385,14 @@ public Connection getOrConnect(DiscoveryNode node) throws IOException { try (Releasable ignore = connectionLock.acquire(node.getAddress())) { result = tempConnections.get(node.getAddress()); if (result == null) { + ensureOpen(); boolean success = false; result = transportService.openConnection(node, connectionProfile); try { transportService.handshake(result, connectionProfile.getHandshakeTimeout().millis()); synchronized (this) { - // acquire lock to prevent concurrent closing + // acquire lock and check if closed, to prevent leaving an open connection after closing + ensureOpen(); Connection existing = tempConnections.put(node.getAddress(), result); assert existing == null; success = true; From 55099df1cb7f09e878a39f15592201c0f02cae4d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Dec 2016 13:02:39 -0500 Subject: [PATCH 22/22] Support negative numbers in writeVLong (#22314) We don't *want* to use negative numbers with `writeVLong` so throw an exception when we try. On the other hand unforeseen bugs might cause us to write negative numbers (some versions of Elasticsearch don't have the exception, only an assertion) so this fixes `readVLong` so that instead of reading a wrong value and corrupting the stream it reads the negative value. --- .../common/io/stream/StreamInput.java | 17 ++++++--- .../common/io/stream/StreamOutput.java | 18 +++++++--- .../common/io/stream/BytesStreamsTests.java | 36 ++++++++++++++++++- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index f7a8a2ff1a646..f20c372ed10ff 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -214,9 +214,8 @@ public long readLong() throws IOException { } /** - * Reads a long stored in variable-length format. Reads between one and - * nine bytes. Smaller values take fewer bytes. Negative numbers are not - * supported. + * Reads a long stored in variable-length format. Reads between one and ten bytes. Smaller values take fewer bytes. Negative numbers + * are encoded in ten bytes so prefer {@link #readLong()} or {@link #readZLong()} for negative numbers. */ public long readVLong() throws IOException { byte b = readByte(); @@ -260,8 +259,16 @@ public long readVLong() throws IOException { return i; } b = readByte(); - assert (b & 0x80) == 0; - return i | ((b & 0x7FL) << 56); + i |= ((b & 0x7FL) << 56); + if ((b & 0x80) == 0) { + return i; + } + b = readByte(); + if (b != 0 && b != 1) { + throw new IOException("Invalid vlong (" + Integer.toHexString(b) + " << 63) | " + Long.toHexString(i)); + } + i |= ((long) b) << 63; + return i; } public long readZLong() throws IOException { diff --git a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index 3f72c80720247..4d57e7c1b8898 100644 --- a/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/core/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -210,12 +210,22 @@ public void writeLong(long i) throws IOException { } /** - * Writes a non-negative long in a variable-length format. - * Writes between one and nine bytes. Smaller values take fewer bytes. - * Negative numbers are not supported. + * Writes a non-negative long in a variable-length format. Writes between one and ten bytes. Smaller values take fewer bytes. Negative + * numbers use ten bytes and trip assertions (if running in tests) so prefer {@link #writeLong(long)} or {@link #writeZLong(long)} for + * negative numbers. */ public void writeVLong(long i) throws IOException { - assert i >= 0; + if (i < 0) { + throw new IllegalStateException("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + i + "]"); + } + writeVLongNoCheck(i); + } + + /** + * Writes a long in a variable-length format without first checking if it is negative. Package private for testing. Use + * {@link #writeVLong(long)} instead. + */ + void writeVLongNoCheck(long i) throws IOException { while ((i & ~0x7F) != 0) { writeByte((byte) ((i & 0x7f) | 0x80)); i >>>= 7; diff --git a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java index d1340af0b229f..17761d9687f11 100644 --- a/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java +++ b/core/src/test/java/org/elasticsearch/common/io/stream/BytesStreamsTests.java @@ -21,7 +21,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; -import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.GeoPoint; @@ -33,6 +33,7 @@ import java.io.EOFException; import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -778,4 +779,37 @@ public void testReadNegativeArraySize() throws IOException { } } } + + public void testVInt() throws IOException { + final int value = randomInt(); + BytesStreamOutput output = new BytesStreamOutput(); + output.writeVInt(value); + StreamInput input = output.bytes().streamInput(); + assertEquals(value, input.readVInt()); + } + + public void testVLong() throws IOException { + final long value = randomLong(); + { + // Read works for positive and negative numbers + BytesStreamOutput output = new BytesStreamOutput(); + output.writeVLongNoCheck(value); // Use NoCheck variant so we can write negative numbers + StreamInput input = output.bytes().streamInput(); + assertEquals(value, input.readVLong()); + } + if (value < 0) { + // Write doesn't work for negative numbers + BytesStreamOutput output = new BytesStreamOutput(); + Exception e = expectThrows(IllegalStateException.class, () -> output.writeVLong(value)); + assertEquals("Negative longs unsupported, use writeLong or writeZLong for negative numbers [" + value + "]", e.getMessage()); + } + + assertTrue("If we're not compatible with 5.1.1 we can drop the assertion below", + Version.CURRENT.minimumCompatibilityVersion().onOrBefore(Version.V_5_1_1_UNRELEASED)); + /* Read -1 as serialized by a version of Elasticsearch that supported writing negative numbers with writeVLong. Note that this + * should be the same test as the first case (when value is negative) but we've kept some bytes so no matter what we do to + * writeVLong in the future we can be sure we can read bytes as written by Elasticsearch before 5.1.2 */ + StreamInput in = new BytesArray(Base64.getDecoder().decode("////////////AQAAAAAAAA==")).streamInput(); + assertEquals(-1, in.readVLong()); + } }