Skip to content
Permalink
Browse files

Merge branch 'master' into retention-leases-recovery

* master: (23 commits)
  Lift retention lease expiration to index shard (elastic#38380)
  Make Ccr recovery file chunk size configurable (elastic#38370)
  Prevent CCR recovery from missing documents (elastic#38237)
  re-enables awaitsfixed datemath tests (elastic#38376)
  Types removal fix FullClusterRestartIT warnings (elastic#38445)
  Make sure to reject mappings with type _doc when include_type_name is false. (elastic#38270)
  Updates the grok patterns to be consistent with logstash (elastic#27181)
  Ignore type-removal warnings in XPackRestTestHelper (elastic#38431)
  testHlrcFromXContent() should respect assertToXContentEquivalence() (elastic#38232)
  add basic REST test for geohash_grid (elastic#37996)
  Remove DiscoveryPlugin#getDiscoveryTypes (elastic#38414)
  Fix the clock resolution to millis in GetWatchResponseTests (elastic#38405)
  Throw AssertionError when no master (elastic#38432)
  `if_seq_no` and `if_primary_term` parameters aren't wired correctly in REST Client's CRUD API (elastic#38411)
  Enable CronEvalToolTest.testEnsureDateIsShownInRootLocale (elastic#38394)
  Fix failures in BulkProcessorIT#testGlobalParametersAndBulkProcessor. (elastic#38129)
  SQL: Implement CURRENT_DATE (elastic#38175)
  Mute testReadRequestsReturnLatestMappingVersion (elastic#38438)
  [ML] Report index unavailable instead of waiting for lazy node (elastic#38423)
  Update Rollup Caps to allow unknown fields (elastic#38339)
  ...
  • Loading branch information...
jasontedor committed Feb 5, 2019
2 parents 439c1d0 + b03d138 commit d010f37cebf051d58aa9c77517744f6e997c92c4
Showing with 3,824 additions and 3,604 deletions.
  1. +9 −0 client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java
  2. +0 −2 client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutFollowRequest.java
  3. +0 −3 client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/ResumeFollowRequest.java
  4. +1 −1 client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollableIndexCaps.java
  5. +6 −20 client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/RollupJobCaps.java
  6. +35 −27 client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java
  7. +14 −8 client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java
  8. +31 −4 client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java
  9. +2 −3 client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutFollowRequestTests.java
  10. +3 −5 client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/ResumeFollowRequestTests.java
  11. +8 −12 client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java
  12. +6 −14 ...high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java
  13. +10 −0 client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupCapsResponseTests.java
  14. +10 −0 ...est-high-level/src/test/java/org/elasticsearch/client/rollup/GetRollupIndexCapsResponseTests.java
  15. +13 −3 client/rest-high-level/src/test/java/org/elasticsearch/client/rollup/RollupCapsResponseTestCase.java
  16. +152 −181 docs/build.gradle
  17. +5 −0 docs/reference/migration/migrate_7_0/plugins.asciidoc
  18. +77 −2 docs/reference/sql/functions/date-time.asciidoc
  19. +5 −5 libs/grok/src/main/resources/patterns/grok-patterns
  20. +15 −21 modules/lang-painless/src/test/resources/rest-api-spec/test/painless/80_script_score.yml
  21. +2 −3 modules/reindex/src/test/resources/rest-api-spec/test/update_by_query/20_validation.yml
  22. +0 −6 modules/transport-netty4/src/test/java/org/elasticsearch/rest/discovery/Zen2RestApiIT.java
  23. +1 −0 qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java
  24. +42 −0 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml
  25. +45 −0 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/81_cas_with_types.yml
  26. +1 −1 rest-api-spec/src/main/resources/rest-api-spec/test/index/30_cas.yml
  27. +19 −0 rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/10_basic.yml
  28. +23 −0 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/10_basic.yml
  29. +21 −0 rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_template/10_basic.yml
  30. +29 −0 rest-api-spec/src/main/resources/rest-api-spec/test/indices.rollover/40_mapping.yml
  31. +62 −0 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml
  32. +7 −2 server/src/main/java/org/elasticsearch/action/admin/indices/rollover/RolloverRequest.java
  33. +12 −7 server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java
  34. +32 −4 server/src/main/java/org/elasticsearch/cluster/SnapshotsInProgress.java
  35. +5 −15 server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java
  36. +4 −11 server/src/main/java/org/elasticsearch/discovery/DiscoveryModule.java
  37. +11 −11 server/src/main/java/org/elasticsearch/index/IndexService.java
  38. +5 −0 server/src/main/java/org/elasticsearch/index/IndexSettings.java
  39. +15 −0 server/src/main/java/org/elasticsearch/index/mapper/MapperService.java
  40. +38 −38 server/src/main/java/org/elasticsearch/index/seqno/ReplicationTracker.java
  41. +22 −4 server/src/main/java/org/elasticsearch/index/shard/IndexShard.java
  42. +4 −4 server/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java
  43. +6 −5 server/src/main/java/org/elasticsearch/index/store/Store.java
  44. +7 −1 server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java
  45. +5 −39 server/src/main/java/org/elasticsearch/plugins/DiscoveryPlugin.java
  46. +26 −6 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexAction.java
  47. +5 −16 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateAction.java
  48. +12 −4 server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestPutMappingAction.java
  49. +0 −2 server/src/main/java/org/elasticsearch/snapshots/SnapshotShardsService.java
  50. +199 −237 server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java
  51. +9 −10 server/src/test/java/org/elasticsearch/action/admin/indices/rollover/RolloverIT.java
  52. +0 −7 server/src/test/java/org/elasticsearch/action/support/master/IndexingMasterFailoverIT.java
  53. +1 −4 server/src/test/java/org/elasticsearch/client/transport/TransportClientIT.java
  54. +0 −1 server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
  55. +0 −7 server/src/test/java/org/elasticsearch/cluster/coordination/RareClusterStateIT.java
  56. +3 −5 server/src/test/java/org/elasticsearch/cluster/coordination/Zen1IT.java
  57. +0 −3 server/src/test/java/org/elasticsearch/cluster/metadata/DateMathExpressionResolverTests.java
  58. +0 −7 server/src/test/java/org/elasticsearch/cluster/routing/PrimaryAllocationIT.java
  59. +1 −18 server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java
  60. +0 −35 server/src/test/java/org/elasticsearch/discovery/DiscoveryModuleTests.java
  61. +3 −10 server/src/test/java/org/elasticsearch/discovery/SnapshotDisruptionIT.java
  62. +24 −110 server/src/test/java/org/elasticsearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java
  63. +28 −18 server/src/test/java/org/elasticsearch/index/seqno/RetentionLeaseSyncIT.java
  64. +38 −31 server/src/test/java/org/elasticsearch/index/shard/IndexShardRetentionLeaseTests.java
  65. +12 −119 server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
  66. +29 −2 server/src/test/java/org/elasticsearch/ingest/PipelineConfigurationTests.java
  67. +80 −0 server/src/test/java/org/elasticsearch/rest/action/admin/indices/RestCreateIndexActionTests.java
  68. +1 −67 ...er/src/test/java/org/elasticsearch/rest/action/admin/indices/RestPutIndexTemplateActionTests.java
  69. +0 −1 server/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
  70. +3 −9 server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
  71. +146 −0 test/framework/src/main/java/org/elasticsearch/index/shard/RestoreOnlyRepository.java
  72. +6 −0 test/framework/src/main/java/org/elasticsearch/test/BackgroundIndexer.java
  73. +0 −15 test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java
  74. +0 −6 test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java
  75. +0 −11 test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java
  76. +17 −11 test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java
  77. +0 −133 test/framework/src/main/java/org/elasticsearch/test/discovery/TestZenDiscovery.java
  78. +1 −2 test/framework/src/test/java/org/elasticsearch/test/test/InternalTestClusterTests.java
  79. +97 −104 x-pack/docs/build.gradle
  80. +20 −0 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CcrSettings.java
  81. +11 −15 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
  82. +12 −13 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportFollowInfoAction.java
  83. +11 −12 ...gin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutAutoFollowPatternAction.java
  84. +17 −16 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
  85. +28 −25 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
  86. +1 −3 x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java
  87. +74 −5 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java
  88. +5 −3 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrSingleNodeTestCase.java
  89. +37 −32 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java
  90. +10 −1 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java
  91. +16 −15 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java
  92. +70 −78 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java
  93. +2 −2 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/LocalIndexFollowingIT.java
  94. +3 −3 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinatorTests.java
  95. +16 −63 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/FollowInfoResponseTests.java
  96. +18 −36 ...plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutAutoFollowPatternRequestTests.java
  97. +19 −2 x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/PutFollowActionRequestTests.java
  98. +30 −18 ...k/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ResumeFollowActionRequestTests.java
  99. +78 −0 ...lugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowEngineIndexShardTests.java
  100. +2 −1 ...plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/repository/CcrRestoreSourceServiceTests.java
  101. +2 −1 x-pack/plugin/core/src/main/java/org/elasticsearch/snapshots/SourceOnlySnapshotRepository.java
  102. +2 −156 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowInfoAction.java
  103. +314 −0 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/FollowParameters.java
  104. +73 −232 ...plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutAutoFollowPatternAction.java
  105. +54 −88 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/PutFollowAction.java
  106. +26 −271 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/action/ResumeFollowAction.java
  107. +1 −1 x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/AbstractHlrcXContentTestCase.java
  108. +13 −7 x-pack/plugin/core/src/test/java/org/elasticsearch/protocol/xpack/watcher/GetWatchResponseTests.java
  109. +4 −1 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestTestHelper.java
  110. +1 −1 ...lm/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/indexlifecycle/CCRIndexLifecycleIT.java
  111. +28 −26 x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
  112. +56 −20 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
  113. +0 −9 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/NetworkDisruptionIT.java
  114. +1 −2 x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java
  115. +1 −18 x-pack/plugin/security/src/test/java/org/elasticsearch/license/LicensingTests.java
  116. +5 −22 ...c/test/java/org/elasticsearch/xpack/security/transport/ServerTransportFilterIntegrationTests.java
  117. +1 −0 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/cli/ShowTestCase.java
  118. +50 −62 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
  119. +4 −8 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/FetchSizeTestCase.java
  120. +5 −1 x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
  121. +87 −0 x-pack/plugin/sql/qa/src/main/resources/date.csv-spec
  122. +7 −0 x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec
  123. +62 −12 x-pack/plugin/sql/qa/src/main/resources/docs.csv-spec
  124. +4 −3 x-pack/plugin/sql/src/main/antlr/SqlBase.g4
  125. +2 −2 x-pack/plugin/sql/src/main/antlr/SqlBase.tokens
  126. +2 −2 x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens
  127. +3 −1 ...ck/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java
  128. +1 −1 ...l/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/ConfigurationFunction.java
  129. +25 −0 ...ql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDate.java
  130. +14 −38 ...rc/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTime.java
  131. +49 −0 ...rc/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentFunction.java
  132. +6 −4 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java
  133. +362 −361 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java
  134. +338 −318 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java
  135. +46 −0 ...c/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTests.java
  136. +51 −13 ...st/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/CurrentDateTimeTests.java
  137. +17 −1 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/ExpressionTests.java
  138. +2 −5 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/tree/NodeSubclassTests.java
  139. +5 −6 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/delete_job.yml
  140. +5 −6 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/get_jobs.yml
  141. +5 −6 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml
  142. +21 −44 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml
  143. +17 −24 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/security_tests.yml
  144. +5 −6 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/start_job.yml
  145. +5 −6 x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/stop_job.yml
  146. +0 −1 ...atcher/src/test/java/org/elasticsearch/xpack/watcher/trigger/schedule/tool/CronEvalToolTests.java
  147. +1 −2 x-pack/qa/multi-node/src/test/java/org/elasticsearch/multi_node/RollupIT.java
@@ -108,6 +108,8 @@ static Request delete(DeleteRequest deleteRequest) {
parameters.withTimeout(deleteRequest.timeout());
parameters.withVersion(deleteRequest.version());
parameters.withVersionType(deleteRequest.versionType());
parameters.withIfSeqNo(deleteRequest.ifSeqNo());
parameters.withIfPrimaryTerm(deleteRequest.ifPrimaryTerm());
parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy());
parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards());
return request;
@@ -191,6 +193,11 @@ static Request bulk(BulkRequest bulkRequest) throws IOException {
}
}

if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
metadata.field("if_seq_no", action.ifSeqNo());
metadata.field("if_primary_term", action.ifPrimaryTerm());
}

if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
IndexRequest indexRequest = (IndexRequest) action;
if (Strings.hasLength(indexRequest.getPipeline())) {
@@ -319,6 +326,8 @@ static Request index(IndexRequest indexRequest) {
parameters.withTimeout(indexRequest.timeout());
parameters.withVersion(indexRequest.version());
parameters.withVersionType(indexRequest.versionType());
parameters.withIfSeqNo(indexRequest.ifSeqNo());
parameters.withIfPrimaryTerm(indexRequest.ifPrimaryTerm());
parameters.withPipeline(indexRequest.getPipeline());
parameters.withRefreshPolicy(indexRequest.getRefreshPolicy());
parameters.withWaitForActiveShards(indexRequest.waitForActiveShards());
@@ -32,7 +32,6 @@

static final ParseField REMOTE_CLUSTER_FIELD = new ParseField("remote_cluster");
static final ParseField LEADER_INDEX_FIELD = new ParseField("leader_index");
static final ParseField FOLLOWER_INDEX_FIELD = new ParseField("follower_index");

private final String remoteCluster;
private final String leaderIndex;
@@ -55,7 +54,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.startObject();
builder.field(REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
builder.field(LEADER_INDEX_FIELD.getPreferredName(), leaderIndex);
builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex);
toXContentFragment(builder, params);
builder.endObject();
return builder;
@@ -26,8 +26,6 @@
import java.io.IOException;
import java.util.Objects;

import static org.elasticsearch.client.ccr.PutFollowRequest.FOLLOWER_INDEX_FIELD;

public final class ResumeFollowRequest extends FollowConfig implements Validatable, ToXContentObject {

private final String followerIndex;
@@ -39,7 +37,6 @@ public ResumeFollowRequest(String followerIndex) {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(FOLLOWER_INDEX_FIELD.getPreferredName(), followerIndex);
toXContentFragment(builder, params);
builder.endObject();
return builder;
@@ -44,7 +44,7 @@
public static final Function<String, ConstructingObjectParser<RollableIndexCaps, Void>> PARSER = indexName -> {
@SuppressWarnings("unchecked")
ConstructingObjectParser<RollableIndexCaps, Void> p
= new ConstructingObjectParser<>(indexName,
= new ConstructingObjectParser<>(indexName, true,
a -> new RollableIndexCaps(indexName, (List<RollupJobCaps>) a[0]));

p.declareObjectArray(ConstructingObjectParser.constructorArg(), RollupJobCaps.PARSER::apply,
@@ -33,7 +33,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;

/**
* Represents the Rollup capabilities for a specific job on a single rollup index
@@ -45,15 +45,12 @@
private static final ParseField FIELDS = new ParseField("fields");
private static final String NAME = "rollup_job_caps";

public static final ConstructingObjectParser<RollupJobCaps, Void> PARSER = new ConstructingObjectParser<>(NAME,
public static final ConstructingObjectParser<RollupJobCaps, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
a -> {
@SuppressWarnings("unchecked")
List<Tuple<String, RollupFieldCaps>> caps = (List<Tuple<String, RollupFieldCaps>>) a[3];
if (caps.isEmpty()) {
return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], Collections.emptyMap());
}
Map<String, RollupFieldCaps> mapCaps = new HashMap<>(caps.size());
caps.forEach(c -> mapCaps.put(c.v1(), c.v2()));
Map<String, RollupFieldCaps> mapCaps =
new HashMap<>(caps.stream().collect(Collectors.toMap(Tuple::v1, Tuple::v2)));
return new RollupJobCaps((String) a[0], (String) a[1], (String) a[2], mapCaps);
});

@@ -140,16 +137,6 @@ public int hashCode() {
private static final String NAME = "rollup_field_caps";
private final List<Map<String, Object>> aggs;

public static final Function<String, ConstructingObjectParser<RollupFieldCaps, Void>> PARSER = fieldName -> {
@SuppressWarnings("unchecked")
ConstructingObjectParser<RollupFieldCaps, Void> parser
= new ConstructingObjectParser<>(NAME, a -> new RollupFieldCaps((List<Map<String, Object>>) a[0]));

parser.declareObjectArray(ConstructingObjectParser.constructorArg(),
(p, c) -> p.map(), new ParseField(fieldName));
return parser;
};

RollupFieldCaps(final List<Map<String, Object>> aggs) {
this.aggs = Collections.unmodifiableList(Objects.requireNonNull(aggs));
}
@@ -170,13 +157,12 @@ public static RollupFieldCaps fromXContent(XContentParser parser) throws IOExcep
List<Map<String, Object>> aggs = new ArrayList<>();
if (parser.nextToken().equals(XContentParser.Token.START_ARRAY)) {
while (parser.nextToken().equals(XContentParser.Token.START_OBJECT)) {
aggs.add(Collections.unmodifiableMap(parser.map()));
aggs.add(parser.map());
}
}
return new RollupFieldCaps(Collections.unmodifiableList(aggs));
return new RollupFieldCaps(aggs);
}


@Override
public boolean equals(Object other) {
if (this == other) {
@@ -34,6 +34,7 @@
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.rest.action.document.RestBulkAction;
@@ -75,12 +76,12 @@
(request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT,
bulkListener), listener);
}

private static BulkProcessor.Builder initBulkProcessorBuilderUsingTypes(BulkProcessor.Listener listener) {
return BulkProcessor.builder(
(request, bulkListener) -> highLevelClient().bulkAsync(request, expectWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE),
bulkListener), listener);
}
}

public void testThatBulkProcessorCountIsCorrect() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
@@ -383,22 +384,22 @@ public void testGlobalParametersAndBulkProcessor() throws Exception {
.build()) {
indexDocs(processor, numDocs, null, localType, "test", globalType, "pipeline_id");
latch.await();

assertThat(listener.beforeCounts.get(), equalTo(1));
assertThat(listener.afterCounts.get(), equalTo(1));
assertThat(listener.bulkFailures.size(), equalTo(0));
assertResponseItems(listener.bulkItems, numDocs, localType);

Iterable<SearchHit> hits = searchAll(new SearchRequest("test").routing("routing"));

assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
assertThat(hits, everyItem(Matchers.allOf(hasIndex("test"), hasType(localType))));
assertThat(hits, containsInAnyOrder(expectedIds(numDocs)));
}
}
{
//Check that untyped document additions and untyped global inherit the established custom type
// (the custom document type introduced to the mapping by the earlier code in this test)
// (the custom document type introduced to the mapping by the earlier code in this test)
String globalType = null;
String localType = null;
final CountDownLatch latch = new CountDownLatch(1);
@@ -414,20 +415,19 @@ public void testGlobalParametersAndBulkProcessor() throws Exception {
.build()) {
indexDocs(processor, numDocs, null, localType, "test", globalType, "pipeline_id");
latch.await();

assertThat(listener.beforeCounts.get(), equalTo(1));
assertThat(listener.afterCounts.get(), equalTo(1));
assertThat(listener.bulkFailures.size(), equalTo(0));
assertResponseItems(listener.bulkItems, numDocs, MapperService.SINGLE_MAPPING_NAME);

Iterable<SearchHit> hits = searchAll(new SearchRequest("test").routing("routing"));

assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
assertThat(hits, everyItem(Matchers.allOf(hasIndex("test"), hasType(customType))));
assertThat(hits, containsInAnyOrder(expectedIds(numDocs)));
}
}
assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE);
}
}

@SuppressWarnings("unchecked")
@@ -438,8 +438,8 @@ public void testGlobalParametersAndBulkProcessor() throws Exception {
.<Matcher<SearchHit>>toArray(Matcher[]::new);
}

private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs, String localIndex, String localType,
String globalIndex, String globalType, String globalPipeline) throws Exception {
private MultiGetRequest indexDocs(BulkProcessor processor, int numDocs, String localIndex, String localType,
String globalIndex, String globalType, String globalPipeline) throws Exception {
MultiGetRequest multiGetRequest = new MultiGetRequest();
for (int i = 1; i <= numDocs; i++) {
if (randomBoolean()) {
@@ -448,33 +448,41 @@ private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs, S
} else {
BytesArray data = bytesBulkRequest(localIndex, localType, i);
processor.add(data, globalIndex, globalType, globalPipeline, null, XContentType.JSON);

if (localType != null) {
// If the payload contains types, parsing it into a bulk request results in a warning.
assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE);
}
}
multiGetRequest.add(localIndex, Integer.toString(i));
}
return multiGetRequest;
}

private static BytesArray bytesBulkRequest(String localIndex, String localType, int id) throws IOException {
String action = Strings.toString(jsonBuilder()
.startObject()
.startObject("index")
.field("_index", localIndex)
.field("_type", localType)
.field("_id", Integer.toString(id))
.endObject()
.endObject()
);
String source = Strings.toString(jsonBuilder()
XContentBuilder action = jsonBuilder().startObject().startObject("index");

if (localIndex != null) {
action.field("_index", localIndex);
}

if (localType != null) {
action.field("_type", localType);
}

action.field("_id", Integer.toString(id));
action.endObject().endObject();

XContentBuilder source = jsonBuilder()
.startObject()
.field("field", randomRealisticUnicodeOfLengthBetween(1, 30))
.endObject()
);
.endObject();

String request = action + "\n" + source + "\n";
String request = Strings.toString(action) + "\n" + Strings.toString(source) + "\n";
return new BytesArray(request);
}

private static MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) throws Exception {
private MultiGetRequest indexDocs(BulkProcessor processor, int numDocs) throws Exception {
return indexDocs(processor, numDocs, "test", null, null, null, null);
}

@@ -104,11 +104,13 @@ public void testDelete() throws IOException {
{
// Testing deletion
String docId = "id";
highLevelClient().index(
IndexResponse indexResponse = highLevelClient().index(
new IndexRequest("index").id(docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT);
assertThat(indexResponse.getSeqNo(), greaterThanOrEqualTo(0L));
DeleteRequest deleteRequest = new DeleteRequest("index", docId);
if (randomBoolean()) {
deleteRequest.version(1L);
deleteRequest.setIfSeqNo(indexResponse.getSeqNo());
deleteRequest.setIfPrimaryTerm(indexResponse.getPrimaryTerm());
}
DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync);
assertEquals("index", deleteResponse.getIndex());
@@ -131,12 +133,13 @@ public void testDelete() throws IOException {
String docId = "version_conflict";
highLevelClient().index(
new IndexRequest("index").id( docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT);
DeleteRequest deleteRequest = new DeleteRequest("index", docId).version(2);
DeleteRequest deleteRequest = new DeleteRequest("index", docId).setIfSeqNo(2).setIfPrimaryTerm(2);
ElasticsearchException exception = expectThrows(ElasticsearchException.class,
() -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync));
assertEquals(RestStatus.CONFLICT, exception.status());
assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][" + docId + "]: " +
"version conflict, current version [1] is different than the one provided [2]]", exception.getMessage());
"version conflict, required seqNo [2], primary term [2]. current document has seqNo [3] and primary term [1]]",
exception.getMessage());
assertEquals("index", exception.getMetadata("es.index").get(0));
}
{
@@ -519,13 +522,14 @@ public void testIndex() throws IOException {
ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> {
IndexRequest wrongRequest = new IndexRequest("index").id("id");
wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject());
wrongRequest.version(5L);
wrongRequest.setIfSeqNo(1L).setIfPrimaryTerm(5L);

execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync);
});
assertEquals(RestStatus.CONFLICT, exception.status());
assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[_doc][id]: " +
"version conflict, current version [2] is different than the one provided [5]]", exception.getMessage());
"version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]",
exception.getMessage());
assertEquals("index", exception.getMetadata("es.index").get(0));
}
{
@@ -820,7 +824,8 @@ public void testBulk() throws IOException {
if (opType == DocWriteRequest.OpType.INDEX) {
IndexRequest indexRequest = new IndexRequest("index").id(id).source(source, xContentType);
if (erroneous) {
indexRequest.version(12L);
indexRequest.setIfSeqNo(12L);
indexRequest.setIfPrimaryTerm(12L);
}
bulkRequest.add(indexRequest);

@@ -1130,7 +1135,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure)
if (opType == DocWriteRequest.OpType.INDEX) {
IndexRequest indexRequest = new IndexRequest("index").id(id).source(xContentType, "id", i);
if (erroneous) {
indexRequest.version(12L);
indexRequest.setIfSeqNo(12L);
indexRequest.setIfPrimaryTerm(12L);
}
processor.add(indexRequest);

0 comments on commit d010f37

Please sign in to comment.
You can’t perform that action at this time.