diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index c77ef0f9d5b73..10387db0530e3 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -19,7 +19,6 @@ - @@ -101,7 +100,6 @@ - diff --git a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java index 4df43b75401f9..0925c7441446a 100644 --- a/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java +++ b/core/src/main/java/org/elasticsearch/action/DocWriteResponse.java @@ -18,10 +18,15 @@ */ package org.elasticsearch.action; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.WriteResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.StatusToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.rest.RestStatus; @@ -30,12 +35,13 @@ /** * A base class for the response of a write operation that involves a single doc */ -public abstract class DocWriteResponse extends ReplicationResponse implements StatusToXContent { +public abstract class DocWriteResponse extends ReplicationResponse implements WriteResponse, StatusToXContent { private ShardId shardId; private String id; private String type; private long version; + private boolean forcedRefresh; public DocWriteResponse(ShardId shardId, String type, String id, long version) { this.shardId = shardId; @@ -84,6 +90,20 @@ public long getVersion() { return this.version; } + /** + * Did this request force a refresh? Requests that set {@link WriteRequest#setRefreshPolicy(RefreshPolicy)} to + * {@link RefreshPolicy#IMMEDIATE} will always return true for this. Requests that set it to {@link RefreshPolicy#WAIT_UNTIL} will + * only return true here if they run out of refresh listener slots (see {@link IndexSettings#MAX_REFRESH_LISTENERS_PER_SHARD}). + */ + public boolean forcedRefresh() { + return forcedRefresh; + } + + @Override + public void setForcedRefresh(boolean forcedRefresh) { + this.forcedRefresh = forcedRefresh; + } + /** returns the rest status for this response (based on {@link ShardInfo#status()} */ public RestStatus status() { return getShardInfo().status(); @@ -97,6 +117,7 @@ public void readFrom(StreamInput in) throws IOException { type = in.readString(); id = in.readString(); version = in.readZLong(); + forcedRefresh = in.readBoolean(); } @Override @@ -106,6 +127,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(type); out.writeString(id); out.writeZLong(version); + out.writeBoolean(forcedRefresh); } static final class Fields { @@ -121,7 +143,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields._INDEX, shardId.getIndexName()) .field(Fields._TYPE, type) .field(Fields._ID, id) - .field(Fields._VERSION, version); + .field(Fields._VERSION, version) + .field("forced_refresh", forcedRefresh); shardInfo.toXContent(builder, params); return builder; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index 8bb124d8fc494..a29918b438ef3 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -19,9 +19,9 @@ package org.elasticsearch.action.admin.indices.flush; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java index 7e750b9767790..82fb6d70ca441 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportShardFlushAction.java @@ -19,14 +19,13 @@ package org.elasticsearch.action.admin.indices.flush; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IndexShard; @@ -55,18 +54,19 @@ protected ReplicationResponse newResponseInstance() { } @Override - protected Tuple shardOperationOnPrimary(ShardFlushRequest shardRequest) { + protected PrimaryResult shardOperationOnPrimary(ShardFlushRequest shardRequest) { IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id()); indexShard.flush(shardRequest.getRequest()); logger.trace("{} flush request executed on primary", indexShard.shardId()); - return new Tuple<>(new ReplicationResponse(), shardRequest); + return new PrimaryResult(shardRequest, new ReplicationResponse()); } @Override - protected void shardOperationOnReplica(ShardFlushRequest request) { + protected ReplicaResult shardOperationOnReplica(ShardFlushRequest request) { IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id()); indexShard.flush(request.getRequest()); logger.trace("{} flush request executed on replica", indexShard.shardId()); + return new ReplicaResult(); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index 34bf39daabd19..ac64e276778f6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -19,10 +19,10 @@ package org.elasticsearch.action.admin.indices.refresh; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.BasicReplicationRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.support.replication.TransportBroadcastReplicationAction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java index 0670c1f3cc64d..d7d0c289953a4 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportShardRefreshAction.java @@ -19,15 +19,14 @@ package org.elasticsearch.action.admin.indices.refresh; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.BasicReplicationRequest; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.support.replication.TransportReplicationAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.shard.IndexShard; @@ -36,10 +35,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -/** - * - */ -public class TransportShardRefreshAction extends TransportReplicationAction { +public class TransportShardRefreshAction + extends TransportReplicationAction { public static final String NAME = RefreshAction.NAME + "[s]"; @@ -47,8 +44,8 @@ public class TransportShardRefreshAction extends TransportReplicationAction shardOperationOnPrimary(BasicReplicationRequest shardRequest) { + protected PrimaryResult shardOperationOnPrimary(BasicReplicationRequest shardRequest) { IndexShard indexShard = indicesService.indexServiceSafe(shardRequest.shardId().getIndex()).getShard(shardRequest.shardId().id()); indexShard.refresh("api"); logger.trace("{} refresh request executed on primary", indexShard.shardId()); - return new Tuple<>(new ReplicationResponse(), shardRequest); + return new PrimaryResult(shardRequest, new ReplicationResponse()); } @Override - protected void shardOperationOnReplica(BasicReplicationRequest request) { + protected ReplicaResult shardOperationOnReplica(BasicReplicationRequest request) { final ShardId shardId = request.shardId(); IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); indexShard.refresh("api"); logger.trace("{} refresh request executed on replica", indexShard.shardId()); + return new ReplicaResult(); } @Override diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index a5775656475e4..85d7147ada084 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; @@ -54,16 +55,21 @@ * Note that we only support refresh on the bulk request not per item. * @see org.elasticsearch.client.Client#bulk(BulkRequest) */ -public class BulkRequest extends ActionRequest implements CompositeIndicesRequest { +public class BulkRequest extends ActionRequest implements CompositeIndicesRequest, WriteRequest { private static final int REQUEST_OVERHEAD = 50; + /** + * Requests that are part of this request. It is only possible to add things that are both {@link ActionRequest}s and + * {@link WriteRequest}s to this but java doesn't support syntax to declare that everything in the array has both types so we declare + * the one with the least casts. + */ final List> requests = new ArrayList<>(); List payloads = null; protected TimeValue timeout = BulkShardRequest.DEFAULT_TIMEOUT; private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT; - private boolean refresh = false; + private RefreshPolicy refreshPolicy = RefreshPolicy.NONE; private long sizeInBytes = 0; @@ -437,18 +443,15 @@ public WriteConsistencyLevel consistencyLevel() { return this.consistencyLevel; } - /** - * Should a refresh be executed post this bulk operation causing the operations to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public BulkRequest refresh(boolean refresh) { - this.refresh = refresh; + @Override + public BulkRequest setRefreshPolicy(RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; return this; } - public boolean refresh() { - return this.refresh; + @Override + public RefreshPolicy getRefreshPolicy() { + return refreshPolicy; } /** @@ -483,7 +486,7 @@ private int findNextMarker(byte marker, int from, BytesReference data, int lengt * @return Whether this bulk request contains index request with an ingest pipeline enabled. */ public boolean hasIndexRequestsWithPipelines() { - for (ActionRequest actionRequest : requests) { + for (ActionRequest actionRequest : requests) { if (actionRequest instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) actionRequest; if (Strings.hasText(indexRequest.getPipeline())) { @@ -503,10 +506,9 @@ public ActionRequestValidationException validate() { } for (ActionRequest request : requests) { // We first check if refresh has been set - if ((request instanceof DeleteRequest && ((DeleteRequest)request).refresh()) || - (request instanceof UpdateRequest && ((UpdateRequest)request).refresh()) || - (request instanceof IndexRequest && ((IndexRequest)request).refresh())) { - validationException = addValidationError("Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", validationException); + if (((WriteRequest) request).getRefreshPolicy() != RefreshPolicy.NONE) { + validationException = addValidationError( + "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", validationException); } ActionRequestValidationException ex = request.validate(); if (ex != null) { @@ -541,7 +543,7 @@ public void readFrom(StreamInput in) throws IOException { requests.add(request); } } - refresh = in.readBoolean(); + refreshPolicy = RefreshPolicy.readFrom(in); timeout = TimeValue.readTimeValue(in); } @@ -560,7 +562,7 @@ public void writeTo(StreamOutput out) throws IOException { } request.writeTo(out); } - out.writeBoolean(refresh); + refreshPolicy.writeTo(out); timeout.writeTo(out); } } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java index 3744055d26cdb..4f2b7aa702ecf 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkRequestBuilder.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.delete.DeleteRequestBuilder; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; @@ -35,7 +36,8 @@ * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes * it in a single batch. */ -public class BulkRequestBuilder extends ActionRequestBuilder { +public class BulkRequestBuilder extends ActionRequestBuilder + implements WriteRequestBuilder { public BulkRequestBuilder(ElasticsearchClient client, BulkAction action) { super(client, action, new BulkRequest()); @@ -116,16 +118,6 @@ public BulkRequestBuilder setConsistencyLevel(WriteConsistencyLevel consistencyL return this; } - /** - * Should a refresh be executed post this bulk operation causing the operations to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public BulkRequestBuilder setRefresh(boolean refresh) { - request.refresh(refresh); - return this; - } - /** * A timeout to wait if the index operation can't be performed immediately. Defaults to 1m. */ diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java index 874789e8d61f4..321b7e2a8e505 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardRequest.java @@ -19,7 +19,7 @@ package org.elasticsearch.action.bulk; -import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; @@ -31,23 +31,17 @@ /** * */ -public class BulkShardRequest extends ReplicationRequest { +public class BulkShardRequest extends ReplicatedWriteRequest { private BulkItemRequest[] items; - private boolean refresh; - public BulkShardRequest() { } - BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, boolean refresh, BulkItemRequest[] items) { + BulkShardRequest(BulkRequest bulkRequest, ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) { super(shardId); this.items = items; - this.refresh = refresh; - } - - boolean refresh() { - return this.refresh; + setRefreshPolicy(refreshPolicy); } BulkItemRequest[] items() { @@ -77,7 +71,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - out.writeBoolean(refresh); } @Override @@ -89,7 +82,6 @@ public void readFrom(StreamInput in) throws IOException { items[i] = BulkItemRequest.readBulkItem(in); } } - refresh = in.readBoolean(); } @Override @@ -97,8 +89,15 @@ public String toString() { // This is included in error messages so we'll try to make it somewhat user friendly. StringBuilder b = new StringBuilder("BulkShardRequest to ["); b.append(index).append("] containing [").append(items.length).append("] requests"); - if (refresh) { + switch (getRefreshPolicy()) { + case IMMEDIATE: b.append(" and a refresh"); + break; + case WAIT_UNTIL: + b.append(" blocking until refresh"); + break; + case NONE: + break; } return b.toString(); } diff --git a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java index 76c80a9b0640a..22260181bb175 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/BulkShardResponse.java @@ -19,7 +19,9 @@ package org.elasticsearch.action.bulk; -import org.elasticsearch.action.ReplicationResponse; +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.support.WriteResponse; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; @@ -29,7 +31,7 @@ /** * */ -public class BulkShardResponse extends ReplicationResponse { +public class BulkShardResponse extends ReplicationResponse implements WriteResponse { private ShardId shardId; private BulkItemResponse[] responses; @@ -50,6 +52,20 @@ public BulkItemResponse[] getResponses() { return responses; } + @Override + public void setForcedRefresh(boolean forcedRefresh) { + /* + * Each DocWriteResponse already has a location for whether or not it forced a refresh so we just set that information on the + * response. + */ + for (BulkItemResponse response : responses) { + DocWriteResponse r = response.getResponse(); + if (r != null) { + r.setForcedRefresh(forcedRefresh); + } + } + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 667e691f6c86e..4cbebd0739a1e 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -344,7 +344,8 @@ void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeN for (Map.Entry> entry : requestsByShard.entrySet()) { final ShardId shardId = entry.getKey(); final List requests = entry.getValue(); - BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); + BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.getRefreshPolicy(), + requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); bulkShardRequest.timeout(bulkRequest.timeout()); if (task != null) { diff --git a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index a2f642374b74e..4ad1136e668c1 100644 --- a/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/core/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -30,7 +30,8 @@ import org.elasticsearch.action.index.TransportIndexAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.replication.ReplicationRequest; -import org.elasticsearch.action.support.replication.TransportReplicationAction; +import org.elasticsearch.action.support.replication.TransportWriteAction; +import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; import org.elasticsearch.action.update.UpdateHelper; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; @@ -53,6 +54,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.Translog.Location; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.threadpool.ThreadPool; @@ -67,7 +69,7 @@ /** * Performs the index operation. */ -public class TransportShardBulkAction extends TransportReplicationAction { +public class TransportShardBulkAction extends TransportWriteAction { private final static String OP_TYPE_UPDATE = "update"; private final static String OP_TYPE_DELETE = "delete"; @@ -83,9 +85,8 @@ public TransportShardBulkAction(Settings settings, TransportService transportSer IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction, MappingUpdatedAction mappingUpdatedAction, UpdateHelper updateHelper, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { - super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, - actionFilters, indexNameExpressionResolver, - BulkShardRequest::new, BulkShardRequest::new, ThreadPool.Names.BULK); + super(settings, ACTION_NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, + indexNameExpressionResolver, BulkShardRequest::new, ThreadPool.Names.BULK); this.updateHelper = updateHelper; this.allowIdGeneration = settings.getAsBoolean("action.allow_id_generation", true); this.mappingUpdatedAction = mappingUpdatedAction; @@ -107,10 +108,9 @@ protected boolean resolveIndex() { } @Override - protected Tuple shardOperationOnPrimary(BulkShardRequest request) { + protected WriteResult onPrimaryShard(BulkShardRequest request, IndexShard indexShard) throws Exception { ShardId shardId = request.shardId(); final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - final IndexShard indexShard = indexService.getShard(shardId.getId()); final IndexMetaData metaData = indexService.getIndexSettings().getIndexMetaData(); long[] preVersions = new long[request.items().length]; @@ -121,13 +121,13 @@ protected Tuple shardOperationOnPrimary(Bul location = handleItem(metaData, request, indexShard, preVersions, preVersionTypes, location, requestIndex, item); } - processAfterWrite(request.refresh(), indexShard, location); BulkItemResponse[] responses = new BulkItemResponse[request.items().length]; BulkItemRequest[] items = request.items(); for (int i = 0; i < items.length; i++) { responses[i] = items[i].getPrimaryResponse(); } - return new Tuple<>(new BulkShardResponse(request.shardId(), responses), request); + BulkShardResponse response = new BulkShardResponse(request.shardId(), responses); + return new WriteResult<>(response, location); } private Translog.Location handleItem(IndexMetaData metaData, BulkShardRequest request, IndexShard indexShard, long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex, BulkItemRequest item) { @@ -154,9 +154,9 @@ private Translog.Location index(IndexMetaData metaData, BulkShardRequest request preVersionTypes[requestIndex] = indexRequest.versionType(); try { WriteResult result = shardIndexOperation(request, indexRequest, metaData, indexShard, true); - location = locationToSync(location, result.location); + location = locationToSync(location, result.getLocation()); // add the response - IndexResponse indexResponse = result.response(); + IndexResponse indexResponse = result.getResponse(); setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), indexResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to handle it @@ -197,8 +197,8 @@ private Translog.Location delete(BulkShardRequest request, IndexShard indexShard try { // add the response final WriteResult writeResult = TransportDeleteAction.executeDeleteRequestOnPrimary(deleteRequest, indexShard); - DeleteResponse deleteResponse = writeResult.response(); - location = locationToSync(location, writeResult.location); + DeleteResponse deleteResponse = writeResult.getResponse(); + location = locationToSync(location, writeResult.getLocation()); setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_DELETE, deleteResponse)); } catch (Throwable e) { // rethrow the failure if we are going to retry on primary and let parent failure to handle it @@ -237,16 +237,17 @@ private Tuple update(IndexMetaData metaData, } if (updateResult.success()) { if (updateResult.writeResult != null) { - location = locationToSync(location, updateResult.writeResult.location); + location = locationToSync(location, updateResult.writeResult.getLocation()); } switch (updateResult.result.operation()) { case UPSERT: case INDEX: + @SuppressWarnings("unchecked") WriteResult result = updateResult.writeResult; IndexRequest indexRequest = updateResult.request(); BytesReference indexSourceAsBytes = indexRequest.source(); // add the response - IndexResponse indexResponse = result.response(); + IndexResponse indexResponse = result.getResponse(); UpdateResponse updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getVersion(), indexResponse.isCreated()); if (updateRequest.fields() != null && updateRequest.fields().length > 0) { Tuple> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true); @@ -256,8 +257,9 @@ private Tuple update(IndexMetaData metaData, setResponse(item, new BulkItemResponse(item.id(), OP_TYPE_UPDATE, updateResponse)); break; case DELETE: + @SuppressWarnings("unchecked") WriteResult writeResult = updateResult.writeResult; - DeleteResponse response = writeResult.response(); + DeleteResponse response = writeResult.getResponse(); DeleteRequest deleteRequest = updateResult.request(); updateResponse = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), response.getVersion(), updateResult.result.updatedSourceAsMap(), updateResult.result.updateSourceContentType(), null)); @@ -326,11 +328,14 @@ private void setResponse(BulkItemRequest request, BulkItemResponse response) { request.setPrimaryResponse(response); if (response.isFailed()) { request.setIgnoreOnReplica(); + } else { + // Set the ShardInfo to 0 so we can safely send it to the replicas. We won't use it in the real response though. + response.getResponse().setShardInfo(new ShardInfo()); } } - private WriteResult shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData, - IndexShard indexShard, boolean processed) throws Throwable { + private WriteResult shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, IndexMetaData metaData, + IndexShard indexShard, boolean processed) throws Throwable { MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type()); if (!processed) { @@ -431,12 +436,8 @@ private UpdateResult shardUpdateOperation(IndexMetaData metaData, BulkShardReque } } - @Override - protected void shardOperationOnReplica(BulkShardRequest request) { - final ShardId shardId = request.shardId(); - IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - IndexShard indexShard = indexService.getShard(shardId.id()); + protected Location onReplicaShard(BulkShardRequest request, IndexShard indexShard) { Translog.Location location = null; for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; @@ -472,8 +473,7 @@ protected void shardOperationOnReplica(BulkShardRequest request) { throw new IllegalStateException("Unexpected index operation: " + item.request()); } } - - processAfterWrite(request.refresh(), indexShard, location); + return location; } private void applyVersion(BulkItemRequest item, long version, VersionType versionType) { diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index cbd10553522f1..bdf09e3e532fd 100644 --- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -21,7 +21,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocumentRequest; -import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -43,7 +43,7 @@ * @see org.elasticsearch.client.Client#delete(DeleteRequest) * @see org.elasticsearch.client.Requests#deleteRequest(String) */ -public class DeleteRequest extends ReplicationRequest implements DocumentRequest { +public class DeleteRequest extends ReplicatedWriteRequest implements DocumentRequest { private String type; private String id; @@ -51,7 +51,6 @@ public class DeleteRequest extends ReplicationRequest implements private String routing; @Nullable private String parent; - private boolean refresh; private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; @@ -165,20 +164,6 @@ public String routing() { return this.routing; } - /** - * Should a refresh be executed post this index operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public DeleteRequest refresh(boolean refresh) { - this.refresh = refresh; - return this; - } - - public boolean refresh() { - return this.refresh; - } - /** * Sets the version, which will cause the delete operation to only be performed if a matching * version exists and no changes happened on the doc since then. @@ -208,7 +193,6 @@ public void readFrom(StreamInput in) throws IOException { id = in.readString(); routing = in.readOptionalString(); parent = in.readOptionalString(); - refresh = in.readBoolean(); version = in.readLong(); versionType = VersionType.fromValue(in.readByte()); } @@ -220,7 +204,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(id); out.writeOptionalString(routing()); out.writeOptionalString(parent()); - out.writeBoolean(refresh); out.writeLong(version); out.writeByte(versionType.getValue()); } diff --git a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java index 0ce907bac1d64..b9b0f95f8de90 100644 --- a/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/delete/DeleteRequestBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.delete; +import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; @@ -27,7 +28,8 @@ /** * A delete document action request builder. */ -public class DeleteRequestBuilder extends ReplicationRequestBuilder { +public class DeleteRequestBuilder extends ReplicationRequestBuilder + implements WriteRequestBuilder { public DeleteRequestBuilder(ElasticsearchClient client, DeleteAction action) { super(client, action, new DeleteRequest()); @@ -71,16 +73,6 @@ public DeleteRequestBuilder setRouting(String routing) { return this; } - /** - * Should a refresh be executed post this index operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public DeleteRequestBuilder setRefresh(boolean refresh) { - request.refresh(refresh); - return this; - } - /** * Sets the version, which will cause the delete operation to only be performed if a matching * version exists and no changes happened on the doc since then. diff --git a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java index 62d46766c4724..beced23c338a8 100644 --- a/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java +++ b/core/src/main/java/org/elasticsearch/action/delete/TransportDeleteAction.java @@ -27,19 +27,19 @@ import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; -import org.elasticsearch.action.support.replication.TransportReplicationAction; +import org.elasticsearch.action.support.replication.TransportWriteAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.shard.ShardStateAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog.Location; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; @@ -49,7 +49,7 @@ /** * Performs the delete operation. */ -public class TransportDeleteAction extends TransportReplicationAction { +public class TransportDeleteAction extends TransportWriteAction { private final AutoCreateIndex autoCreateIndex; private final TransportCreateIndexAction createIndexAction; @@ -60,9 +60,8 @@ public TransportDeleteAction(Settings settings, TransportService transportServic TransportCreateIndexAction createIndexAction, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex) { - super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, - actionFilters, indexNameExpressionResolver, - DeleteRequest::new, DeleteRequest::new, ThreadPool.Names.INDEX); + super(settings, DeleteAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, + indexNameExpressionResolver, DeleteRequest::new, ThreadPool.Names.INDEX); this.createIndexAction = createIndexAction; this.autoCreateIndex = autoCreateIndex; } @@ -119,11 +118,13 @@ protected DeleteResponse newResponseInstance() { } @Override - protected Tuple shardOperationOnPrimary(DeleteRequest request) { - IndexShard indexShard = indicesService.indexServiceSafe(request.shardId().getIndex()).getShard(request.shardId().id()); - final WriteResult result = executeDeleteRequestOnPrimary(request, indexShard); - processAfterWrite(request.refresh(), indexShard, result.location); - return new Tuple<>(result.response, request); + protected WriteResult onPrimaryShard(DeleteRequest request, IndexShard indexShard) { + return executeDeleteRequestOnPrimary(request, indexShard); + } + + @Override + protected Location onReplicaShard(DeleteRequest request, IndexShard indexShard) { + return executeDeleteRequestOnReplica(request, indexShard).getTranslogLocation(); } public static WriteResult executeDeleteRequestOnPrimary(DeleteRequest request, IndexShard indexShard) { @@ -134,9 +135,8 @@ public static WriteResult executeDeleteRequestOnPrimary(DeleteRe request.version(delete.version()); assert request.versionType().validateVersionForWrites(request.version()); - return new WriteResult<>( - new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found()), - delete.getTranslogLocation()); + DeleteResponse response = new DeleteResponse(indexShard.shardId(), request.type(), request.id(), delete.version(), delete.found()); + return new WriteResult<>(response, delete.getTranslogLocation()); } public static Engine.Delete executeDeleteRequestOnReplica(DeleteRequest request, IndexShard indexShard) { @@ -144,13 +144,4 @@ public static Engine.Delete executeDeleteRequestOnReplica(DeleteRequest request, indexShard.delete(delete); return delete; } - - @Override - protected void shardOperationOnReplica(DeleteRequest request) { - final ShardId shardId = request.shardId(); - IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShard(shardId.id()); - Engine.Delete delete = executeDeleteRequestOnReplica(request, indexShard); - processAfterWrite(request.refresh(), indexShard, delete.getTranslogLocation()); - } - } diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 31accebc3b2ad..bc1e631e559ff 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -24,7 +24,7 @@ import org.elasticsearch.action.DocumentRequest; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.TimestampParsingException; -import org.elasticsearch.action.support.replication.ReplicationRequest; +import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; import org.elasticsearch.client.Requests; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; @@ -67,7 +67,7 @@ * @see org.elasticsearch.client.Requests#indexRequest(String) * @see org.elasticsearch.client.Client#index(IndexRequest) */ -public class IndexRequest extends ReplicationRequest implements DocumentRequest { +public class IndexRequest extends ReplicatedWriteRequest implements DocumentRequest { /** * Operation type controls if the type of the index operation. @@ -145,7 +145,6 @@ public static OpType fromString(String sOpType) { private OpType opType = OpType.INDEX; - private boolean refresh = false; private long version = Versions.MATCH_ANY; private VersionType versionType = VersionType.INTERNAL; @@ -542,20 +541,6 @@ public OpType opType() { return this.opType; } - /** - * Should a refresh be executed post this index operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public IndexRequest refresh(boolean refresh) { - this.refresh = refresh; - return this; - } - - public boolean refresh() { - return this.refresh; - } - /** * Sets the version, which will cause the index operation to only be performed if a matching * version exists and no changes happened on the doc since then. @@ -652,7 +637,6 @@ public void readFrom(StreamInput in) throws IOException { source = in.readBytesReference(); opType = OpType.fromId(in.readByte()); - refresh = in.readBoolean(); version = in.readLong(); versionType = VersionType.fromValue(in.readByte()); pipeline = in.readOptionalString(); @@ -674,7 +658,6 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeBytesReference(source); out.writeByte(opType.id()); - out.writeBoolean(refresh); out.writeLong(version); out.writeByte(versionType.getValue()); out.writeOptionalString(pipeline); diff --git a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java index 4116755e4eb05..20587bf0ea99d 100644 --- a/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/index/IndexRequestBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.index; +import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.replication.ReplicationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Nullable; @@ -33,7 +34,8 @@ /** * An index document action request builder. */ -public class IndexRequestBuilder extends ReplicationRequestBuilder { +public class IndexRequestBuilder extends ReplicationRequestBuilder + implements WriteRequestBuilder { public IndexRequestBuilder(ElasticsearchClient client, IndexAction action) { super(client, action, new IndexRequest()); @@ -220,16 +222,6 @@ public IndexRequestBuilder setCreate(boolean create) { return this; } - /** - * Should a refresh be executed post this index operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public IndexRequestBuilder setRefresh(boolean refresh) { - request.refresh(refresh); - return this; - } - /** * Sets the version, which will cause the index operation to only be performed if a matching * version exists and no changes happened on the doc since then. diff --git a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java index 10e18c82b8662..00be64757ae26 100644 --- a/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java +++ b/core/src/main/java/org/elasticsearch/action/index/TransportIndexAction.java @@ -27,7 +27,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.replication.ReplicationOperation; -import org.elasticsearch.action.support.replication.TransportReplicationAction; +import org.elasticsearch.action.support.replication.TransportWriteAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.action.index.MappingUpdatedAction; import org.elasticsearch.cluster.action.shard.ShardStateAction; @@ -36,16 +36,14 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.Translog.Location; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.tasks.Task; @@ -62,7 +60,7 @@ *
  • allowIdGeneration: If the id is set not, should it be generated. Defaults to true. * */ -public class TransportIndexAction extends TransportReplicationAction { +public class TransportIndexAction extends TransportWriteAction { private final AutoCreateIndex autoCreateIndex; private final boolean allowIdGeneration; @@ -78,7 +76,7 @@ public TransportIndexAction(Settings settings, TransportService transportService ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, AutoCreateIndex autoCreateIndex) { super(settings, IndexAction.NAME, transportService, clusterService, indicesService, threadPool, shardStateAction, - actionFilters, indexNameExpressionResolver, IndexRequest::new, IndexRequest::new, ThreadPool.Names.INDEX); + actionFilters, indexNameExpressionResolver, IndexRequest::new, ThreadPool.Names.INDEX); this.mappingUpdatedAction = mappingUpdatedAction; this.createIndexAction = createIndexAction; this.autoCreateIndex = autoCreateIndex; @@ -141,26 +139,13 @@ protected IndexResponse newResponseInstance() { } @Override - protected Tuple shardOperationOnPrimary(IndexRequest request) throws Exception { - - IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); - IndexShard indexShard = indexService.getShard(request.shardId().id()); - - final WriteResult result = executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction); - - final IndexResponse response = result.response; - final Translog.Location location = result.location; - processAfterWrite(request.refresh(), indexShard, location); - return new Tuple<>(response, request); + protected WriteResult onPrimaryShard(IndexRequest request, IndexShard indexShard) throws Exception { + return executeIndexRequestOnPrimary(request, indexShard, mappingUpdatedAction); } @Override - protected void shardOperationOnReplica(IndexRequest request) { - final ShardId shardId = request.shardId(); - IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - IndexShard indexShard = indexService.getShard(shardId.id()); - final Engine.Index operation = executeIndexRequestOnReplica(request, indexShard); - processAfterWrite(request.refresh(), indexShard, operation.getTranslogLocation()); + protected Location onReplicaShard(IndexRequest request, IndexShard indexShard) { + return executeIndexRequestOnReplica(request, indexShard).getTranslogLocation(); } /** @@ -188,11 +173,8 @@ public static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType()); } - /** - * Execute the given {@link IndexRequest} on a primary shard, throwing a - * {@link ReplicationOperation.RetryOnPrimaryException} if the operation needs to be re-tried. - */ - public static WriteResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard, MappingUpdatedAction mappingUpdatedAction) throws Exception { + public static WriteResult executeIndexRequestOnPrimary(IndexRequest request, IndexShard indexShard, + MappingUpdatedAction mappingUpdatedAction) throws Exception { Engine.Index operation = prepareIndexOperationOnPrimary(request, indexShard); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); final ShardId shardId = indexShard.shardId(); @@ -214,8 +196,8 @@ public static WriteResult executeIndexRequestOnPrimary(IndexReque assert request.versionType().validateVersionForWrites(request.version()); - return new WriteResult<>(new IndexResponse(shardId, request.type(), request.id(), request.version(), created), operation.getTranslogLocation()); + IndexResponse response = new IndexResponse(shardId, request.type(), request.id(), request.version(), created); + return new WriteResult<>(response, operation.getTranslogLocation()); } - } diff --git a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java index 1eb9337c814b0..850cac040dd00 100644 --- a/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java +++ b/core/src/main/java/org/elasticsearch/action/ingest/IngestActionFilter.java @@ -162,7 +162,7 @@ BulkRequest getBulkRequest() { return bulkRequest; } else { BulkRequest modifiedBulkRequest = new BulkRequest(); - modifiedBulkRequest.refresh(bulkRequest.refresh()); + modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.consistencyLevel(bulkRequest.consistencyLevel()); modifiedBulkRequest.timeout(bulkRequest.timeout()); diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java new file mode 100644 index 0000000000000..6379a4fb259c3 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequest.java @@ -0,0 +1,109 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.replication.ReplicatedWriteRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Streamable; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; + +/** + * Interface implemented by requests that modify the documents in an index like {@link IndexRequest}, {@link UpdateRequest}, and + * {@link BulkRequest}. Rather than implement this directly most implementers should extend {@link ReplicatedWriteRequest}. + */ +public interface WriteRequest> extends Streamable { + /** + * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh ( + * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default). + */ + R setRefreshPolicy(RefreshPolicy refreshPolicy); + + /** + * Parse the refresh policy from a string, only modifying it if the string is non null. Convenient to use with request parsing. + */ + @SuppressWarnings("unchecked") + default R setRefreshPolicy(String refreshPolicy) { + if (refreshPolicy != null) { + setRefreshPolicy(RefreshPolicy.parse(refreshPolicy)); + } + return (R) this; + } + + /** + * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh ( + * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default). + */ + RefreshPolicy getRefreshPolicy(); + + ActionRequestValidationException validate(); + + enum RefreshPolicy implements Writeable { + /** + * Don't refresh after this request. The default. + */ + NONE, + /** + * Force a refresh as part of this request. This refresh policy does not scale for high indexing or search throughput but is useful + * to present a consistent view to for indices with very low traffic. And it is wonderful for tests! + */ + IMMEDIATE, + /** + * Leave this request open until a refresh has made the contents of this request visible to search. This refresh policy is + * compatible with high indexing and search throughput but it causes the request to wait to reply until a refresh occurs. + */ + WAIT_UNTIL; + + /** + * Parse the string representation of a refresh policy, usually from a request parameter. + */ + public static RefreshPolicy parse(String string) { + switch (string) { + case "false": + return NONE; + /* + * Empty string is IMMEDIATE because that makes "POST /test/test/1?refresh" perform a refresh which reads well and is what folks + * are used to. + */ + case "": + case "true": + return IMMEDIATE; + case "wait_for": + return WAIT_UNTIL; + } + throw new IllegalArgumentException("Unknown value for refresh: [" + string + "]."); + } + + public static RefreshPolicy readFrom(StreamInput in) throws IOException { + return RefreshPolicy.values()[in.readByte()]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeByte((byte) ordinal()); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java new file mode 100644 index 0000000000000..a87fd04345213 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/support/WriteRequestBuilder.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.Version; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; + +public interface WriteRequestBuilder> { + WriteRequest request(); + + /** + * Should this request trigger a refresh ({@linkplain RefreshPolicy#IMMEDIATE}), wait for a refresh ( + * {@linkplain RefreshPolicy#WAIT_UNTIL}), or proceed ignore refreshes entirely ({@linkplain RefreshPolicy#NONE}, the default). + */ + @SuppressWarnings("unchecked") + default B setRefreshPolicy(RefreshPolicy refreshPolicy) { + request().setRefreshPolicy(refreshPolicy); + return (B) this; + } + + /** + * If set to true then this request will force an immediate refresh. Backwards compatibility layer for Elasticsearch's old + * {@code setRefresh} calls. + * + * @deprecated use {@link #setRefreshPolicy(RefreshPolicy)} with {@link RefreshPolicy#IMMEDIATE} or {@link RefreshPolicy#NONE} instead. + * Will be removed in 6.0. + */ + @Deprecated + default B setRefresh(boolean refresh) { + assert Version.CURRENT.major < 6 : "Remove setRefresh(boolean) in 6.0"; + return setRefreshPolicy(refresh ? RefreshPolicy.IMMEDIATE : RefreshPolicy.NONE); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java new file mode 100644 index 0000000000000..07f5ea695d924 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/support/WriteResponse.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support; + +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.index.IndexSettings; + +/** + * Interface implemented by responses for actions that modify the documents in an index like {@link IndexResponse}, {@link UpdateResponse}, + * and {@link BulkResponse}. Rather than implement this directly most implementers should extend {@link DocWriteResponse}. + */ +public interface WriteResponse { + /** + * Mark the response as having forced a refresh? Requests that set {@link WriteRequest#setRefreshPolicy(RefreshPolicy)} to + * {@link RefreshPolicy#IMMEDIATE} should always mark this as true. Requests that set it to {@link RefreshPolicy#WAIT_UNTIL} will only + * set this to true if they run out of refresh listener slots (see {@link IndexSettings#MAX_REFRESH_LISTENERS_PER_SHARD}). + */ + public abstract void setForcedRefresh(boolean forcedRefresh); +} diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java new file mode 100644 index 0000000000000..fa02dac9e1e2d --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicatedWriteRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.replication; + +import org.elasticsearch.action.bulk.BulkShardRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.shard.ShardId; + +import java.io.IOException; + +/** + * Requests that are both {@linkplain ReplicationRequest}s (run on a shard's primary first, then the replica) and {@linkplain WriteRequest} + * (modify documents on a shard), for example {@link BulkShardRequest}, {@link IndexRequest}, and {@link DeleteRequest}. + */ +public abstract class ReplicatedWriteRequest> extends ReplicationRequest implements WriteRequest { + private RefreshPolicy refreshPolicy = RefreshPolicy.NONE; + + /** + * Constructor for deserialization. + */ + public ReplicatedWriteRequest() { + } + + public ReplicatedWriteRequest(ShardId shardId) { + super(shardId); + } + + @Override + @SuppressWarnings("unchecked") + public R setRefreshPolicy(RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; + return (R) this; + } + + @Override + public RefreshPolicy getRefreshPolicy() { + return refreshPolicy; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + refreshPolicy = RefreshPolicy.readFrom(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + refreshPolicy.writeTo(out); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java index 1f7313c19434b..8442e70525783 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationOperation.java @@ -21,7 +21,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.support.TransportActions; @@ -29,7 +28,6 @@ import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.index.engine.VersionConflictEngineException; @@ -47,28 +45,41 @@ import java.util.function.Consumer; import java.util.function.Supplier; -public class ReplicationOperation, ReplicaRequest extends ReplicationRequest, - Response extends ReplicationResponse> { +public class ReplicationOperation< + Request extends ReplicationRequest, + ReplicaRequest extends ReplicationRequest, + PrimaryResultT extends ReplicationOperation.PrimaryResult + > { final private ESLogger logger; final private Request request; final private Supplier clusterStateSupplier; final private String opType; final private AtomicInteger totalShards = new AtomicInteger(); + /** + * The number of pending sub-operations in this operation. This is incremented when the following operations start and decremented when + * they complete: + *
      + *
    • The operation on the primary
      • + *
    • The operation on each replica
      • + *
    • Coordination of the operation as a whole. This prevents the operation from terminating early if we haven't started any replica + * operations and the primary finishes.
      • + *
    + */ final private AtomicInteger pendingShards = new AtomicInteger(); final private AtomicInteger successfulShards = new AtomicInteger(); final private boolean executeOnReplicas; final private boolean checkWriteConsistency; - final private Primary primary; + final private Primary primary; final private Replicas replicasProxy; final private AtomicBoolean finished = new AtomicBoolean(); - final protected ActionListener finalResponseListener; + final protected ActionListener resultListener; - private volatile Response finalResponse = null; + private volatile PrimaryResultT primaryResult = null; private final List shardReplicaFailures = Collections.synchronizedList(new ArrayList<>()); - ReplicationOperation(Request request, Primary primary, - ActionListener listener, + ReplicationOperation(Request request, Primary primary, + ActionListener listener, boolean executeOnReplicas, boolean checkWriteConsistency, Replicas replicas, Supplier clusterStateSupplier, ESLogger logger, String opType) { @@ -76,7 +87,7 @@ public class ReplicationOperation, R this.executeOnReplicas = executeOnReplicas; this.replicasProxy = replicas; this.primary = primary; - this.finalResponseListener = listener; + this.resultListener = listener; this.logger = logger; this.request = request; this.clusterStateSupplier = clusterStateSupplier; @@ -85,28 +96,27 @@ public class ReplicationOperation, R void execute() throws Exception { final String writeConsistencyFailure = checkWriteConsistency ? checkWriteConsistency() : null; - final ShardId shardId = primary.routingEntry().shardId(); + final ShardRouting primaryRouting = primary.routingEntry(); + final ShardId primaryId = primaryRouting.shardId(); if (writeConsistencyFailure != null) { - finishAsFailed(new UnavailableShardsException(shardId, + finishAsFailed(new UnavailableShardsException(primaryId, "{} Timeout: [{}], request: [{}]", writeConsistencyFailure, request.timeout(), request)); return; } totalShards.incrementAndGet(); - pendingShards.incrementAndGet(); // increase by 1 until we finish all primary coordination - Tuple primaryResponse = primary.perform(request); - successfulShards.incrementAndGet(); // mark primary as successful - finalResponse = primaryResponse.v1(); - ReplicaRequest replicaRequest = primaryResponse.v2(); + pendingShards.incrementAndGet(); + primaryResult = primary.perform(request); + final ReplicaRequest replicaRequest = primaryResult.replicaRequest(); assert replicaRequest.primaryTerm() > 0 : "replicaRequest doesn't have a primary term"; if (logger.isTraceEnabled()) { - logger.trace("[{}] op [{}] completed on primary for request [{}]", shardId, opType, request); + logger.trace("[{}] op [{}] completed on primary for request [{}]", primaryId, opType, request); } // we have to get a new state after successfully indexing into the primary in order to honour recovery semantics. // we have to make sure that every operation indexed into the primary after recovery start will also be replicated // to the recovery target. If we use an old cluster state, we may miss a relocation that has started since then. // If the index gets deleted after primary operation, we skip replication - List shards = getShards(shardId, clusterStateSupplier.get()); + final List shards = getShards(primaryId, clusterStateSupplier.get()); final String localNodeId = primary.routingEntry().currentNodeId(); for (final ShardRouting shard : shards) { if (executeOnReplicas == false || shard.unassigned()) { @@ -125,8 +135,8 @@ void execute() throws Exception { } } - // decrement pending and finish (if there are no replicas, or those are done) - decPendingAndFinishIfNeeded(); // incremented in the beginning of this method + successfulShards.incrementAndGet(); + decPendingAndFinishIfNeeded(); } private void performOnReplica(final ShardRouting shard, final ReplicaRequest replicaRequest) { @@ -241,19 +251,19 @@ private void finish() { failuresArray = new ReplicationResponse.ShardInfo.Failure[shardReplicaFailures.size()]; shardReplicaFailures.toArray(failuresArray); } - finalResponse.setShardInfo(new ReplicationResponse.ShardInfo( + primaryResult.setShardInfo(new ReplicationResponse.ShardInfo( totalShards.get(), successfulShards.get(), failuresArray ) ); - finalResponseListener.onResponse(finalResponse); + resultListener.onResponse(primaryResult); } } private void finishAsFailed(Throwable throwable) { if (finished.compareAndSet(false, true)) { - finalResponseListener.onFailure(throwable); + resultListener.onFailure(throwable); } } @@ -284,22 +294,31 @@ public static boolean isConflictException(Throwable e) { } - interface Primary, ReplicaRequest extends ReplicationRequest, - Response extends ReplicationResponse> { + interface Primary< + Request extends ReplicationRequest, + ReplicaRequest extends ReplicationRequest, + PrimaryResultT extends PrimaryResult + > { - /** routing entry for this primary */ + /** + * routing entry for this primary + */ ShardRouting routingEntry(); - /** fail the primary, typically due to the fact that the operation has learned the primary has been demoted by the master */ + /** + * fail the primary, typically due to the fact that the operation has learned the primary has been demoted by the master + */ void failShard(String message, Throwable throwable); /** - * Performs the given request on this primary + * Performs the given request on this primary. Yes, this returns as soon as it can with the request for the replicas and calls a + * listener when the primary request is completed. Yes, the primary request might complete before the method returns. Yes, it might + * also complete after. Deal with it. * - * @return A tuple containing not null values, as first value the result of the primary operation and as second value - * the request to be executed on the replica shards. + * @param request the request to perform + * @return the request to send to the repicas */ - Tuple perform(Request request) throws Exception; + PrimaryResultT perform(Request request) throws Exception; } @@ -308,19 +327,20 @@ interface Replicas> { /** * performs the the given request on the specified replica * - * @param replica {@link ShardRouting} of the shard this request should be executed on + * @param replica {@link ShardRouting} of the shard this request should be executed on * @param replicaRequest operation to peform - * @param listener a callback to call once the operation has been complicated, either successfully or with an error. + * @param listener a callback to call once the operation has been complicated, either successfully or with an error. */ void performOn(ShardRouting replica, ReplicaRequest replicaRequest, ActionListener listener); /** * Fail the specified shard, removing it from the current set of active shards - * @param replica shard to fail - * @param primary the primary shard that requested the failure - * @param message a (short) description of the reason - * @param throwable the original exception which caused the ReplicationOperation to request the shard to be failed - * @param onSuccess a callback to call when the shard has been successfully removed from the active set. + * + * @param replica shard to fail + * @param primary the primary shard that requested the failure + * @param message a (short) description of the reason + * @param throwable the original exception which caused the ReplicationOperation to request the shard to be failed + * @param onSuccess a callback to call when the shard has been successfully removed from the active set. * @param onPrimaryDemoted a callback to call when the shard can not be failed because the current primary has been demoted * by the master. * @param onIgnoredFailure a callback to call when failing a shard has failed, but it that failure can be safely ignored and the @@ -345,4 +365,11 @@ public RetryOnPrimaryException(StreamInput in) throws IOException { super(in); } } + + interface PrimaryResult> { + + R replicaRequest(); + + void setShardInfo(ReplicationResponse.ShardInfo shardInfo); + } } diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java index 3e88575b71789..44c420598b54e 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationRequest.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.WriteConsistencyLevel; +import org.elasticsearch.action.admin.indices.refresh.TransportShardRefreshAction; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; @@ -38,7 +40,8 @@ import static org.elasticsearch.action.ValidateActions.addValidationError; /** - * + * Requests that are run on a particular replica, first on the primary and then on the replicas like {@link IndexRequest} or + * {@link TransportShardRefreshAction}. */ public abstract class ReplicationRequest> extends ActionRequest implements IndicesRequest { @@ -65,7 +68,6 @@ public ReplicationRequest() { } - /** * Creates a new request with resolved shard id */ diff --git a/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java similarity index 96% rename from core/src/main/java/org/elasticsearch/action/ReplicationResponse.java rename to core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java index df2f90b002046..181e0e0ea746c 100644 --- a/core/src/main/java/org/elasticsearch/action/ReplicationResponse.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/ReplicationResponse.java @@ -17,10 +17,12 @@ * under the License. */ -package org.elasticsearch.action; +package org.elasticsearch.action.support.replication; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -79,14 +81,16 @@ public ShardInfo(int total, int successful, Failure... failures) { } /** - * @return the total number of shards the write should go to (replicas and primaries). This includes relocating shards, so this number can be higher than the number of shards. + * @return the total number of shards the write should go to (replicas and primaries). This includes relocating shards, so this + * number can be higher than the number of shards. */ public int getTotal() { return total; } /** - * @return the total number of shards the write succeeded on (replicas and primaries). This includes relocating shards, so this number can be higher than the number of shards. + * @return the total number of shards the write succeeded on (replicas and primaries). This includes relocating shards, so this + * number can be higher than the number of shards. */ public int getSuccessful() { return successful; diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java index 25de821e22714..2cab7d7831795 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportBroadcastReplicationAction.java @@ -22,7 +22,6 @@ import com.carrotsearch.hppc.cursors.IntObjectCursor; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.DefaultShardOperationFailedException; diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java index 8a721dfe50868..87e602bf427fe 100644 --- a/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportReplicationAction.java @@ -22,7 +22,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListenerResponseHandler; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.support.ActionFilters; @@ -41,7 +40,6 @@ import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lease.Releasable; import org.elasticsearch.common.settings.Settings; @@ -53,7 +51,6 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeClosedException; import org.elasticsearch.tasks.Task; @@ -66,6 +63,7 @@ import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportService; import java.io.IOException; @@ -81,9 +79,11 @@ * primary node to validate request before primary operation followed by sampling state again for resolving * nodes with replica copies to perform replication. */ -public abstract class TransportReplicationAction, - ReplicaRequest extends ReplicationRequest, - Response extends ReplicationResponse> extends TransportAction { +public abstract class TransportReplicationAction< + Request extends ReplicationRequest, + ReplicaRequest extends ReplicationRequest, + Response extends ReplicationResponse + > extends TransportAction { final protected TransportService transportService; final protected ClusterService clusterService; @@ -149,17 +149,17 @@ protected void resolveRequest(MetaData metaData, IndexMetaData indexMetaData, Re } /** - * Primary operation on node with primary copy + * Primary operation on node with primary copy. * - * @return A tuple containing not null values, as first value the result of the primary operation and as second value - * the request to be executed on the replica shards. + * @param shardRequest the request to the primary shard */ - protected abstract Tuple shardOperationOnPrimary(Request shardRequest) throws Exception; + protected abstract PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception; /** - * Replica operation on nodes with replica copies + * Synchronous replica operation on nodes with replica copies. This is done under the lock form + * {@link #acquireReplicaOperationLock(ShardId, long)}. */ - protected abstract void shardOperationOnReplica(ReplicaRequest shardRequest); + protected abstract ReplicaResult shardOperationOnReplica(ReplicaRequest shardRequest); /** * True if write consistency should be checked for an implementation @@ -198,26 +198,6 @@ protected boolean retryPrimaryException(Throwable e) { || TransportActions.isShardNotAvailableException(e); } - protected static class WriteResult { - - public final T response; - public final Translog.Location location; - - public WriteResult(T response, Translog.Location location) { - this.response = response; - this.location = location; - } - - @SuppressWarnings("unchecked") - public T response() { - // this sets total, pending and failed to 0 and this is ok, because we will embed this into the replica - // request and not use it - response.setShardInfo(new ReplicationResponse.ShardInfo()); - return (T) response; - } - - } - class OperationTransportHandler implements TransportRequestHandler { @Override public void messageReceived(final Request request, final TransportChannel channel, Task task) throws Exception { @@ -289,7 +269,17 @@ public void handleException(TransportException exp) { final IndexMetaData indexMetaData = clusterService.state().getMetaData().index(request.shardId().getIndex()); final boolean executeOnReplicas = (indexMetaData == null) || shouldExecuteReplication(indexMetaData.getSettings()); final ActionListener listener = createResponseListener(channel, replicationTask, primaryShardReference); - createReplicatedOperation(request, listener, primaryShardReference, executeOnReplicas).execute(); + createReplicatedOperation(request, new ActionListener() { + @Override + public void onResponse(PrimaryResult result) { + result.respond(listener); + } + + @Override + public void onFailure(Throwable e) { + listener.onFailure(e); + } + }, primaryShardReference, executeOnReplicas).execute(); success = true; } } finally { @@ -299,9 +289,9 @@ public void handleException(TransportException exp) { } } - protected ReplicationOperation - createReplicatedOperation(Request request, ActionListener listener, - PrimaryShardReference primaryShardReference, boolean executeOnReplicas) { + protected ReplicationOperation createReplicatedOperation( + Request request, ActionListener listener, + PrimaryShardReference primaryShardReference, boolean executeOnReplicas) { return new ReplicationOperation<>(request, primaryShardReference, listener, executeOnReplicas, checkWriteConsistency(), replicasProxy, clusterService::state, logger, actionName ); @@ -339,6 +329,41 @@ public void onFailure(Throwable e) { } } + protected class PrimaryResult implements ReplicationOperation.PrimaryResult { + final ReplicaRequest replicaRequest; + final Response finalResponse; + + public PrimaryResult(ReplicaRequest replicaRequest, Response finalResponse) { + this.replicaRequest = replicaRequest; + this.finalResponse = finalResponse; + } + + @Override + public ReplicaRequest replicaRequest() { + return replicaRequest; + } + + @Override + public void setShardInfo(ReplicationResponse.ShardInfo shardInfo) { + finalResponse.setShardInfo(shardInfo); + } + + public void respond(ActionListener listener) { + listener.onResponse(finalResponse); + } + } + + protected class ReplicaResult { + /** + * Public constructor so subclasses can call it. + */ + public ReplicaResult() {} + + public void respond(ActionListener listener) { + listener.onResponse(TransportResponse.Empty.INSTANCE); + } + } + class ReplicaOperationTransportHandler implements TransportRequestHandler { @Override public void messageReceived(final ReplicaRequest request, final TransportChannel channel) throws Exception { @@ -426,15 +451,35 @@ protected void responseWithFailure(Throwable t) { protected void doRun() throws Exception { setPhase(task, "replica"); assert request.shardId() != null : "request shardId must be set"; + ReplicaResult result; try (Releasable ignored = acquireReplicaOperationLock(request.shardId(), request.primaryTerm())) { - shardOperationOnReplica(request); + result = shardOperationOnReplica(request); + } + result.respond(new ResponseListener()); + } + + /** + * Listens for the response on the replica and sends the response back to the primary. + */ + private class ResponseListener implements ActionListener { + @Override + public void onResponse(Empty response) { if (logger.isTraceEnabled()) { logger.trace("action [{}] completed on shard [{}] for request [{}]", transportReplicaAction, request.shardId(), - request); + request); + } + setPhase(task, "finished"); + try { + channel.sendResponse(response); + } catch (Exception e) { + onFailure(e); } } - setPhase(task, "finished"); - channel.sendResponse(TransportResponse.Empty.INSTANCE); + + @Override + public void onFailure(Throwable e) { + responseWithFailure(e); + } } } @@ -722,7 +767,7 @@ protected boolean shouldExecuteReplication(Settings settings) { return IndexMetaData.isIndexUsingShadowReplicas(settings) == false; } - class PrimaryShardReference implements ReplicationOperation.Primary, Releasable { + class PrimaryShardReference implements ReplicationOperation.Primary, Releasable { private final IndexShard indexShard; private final Releasable operationLock; @@ -751,9 +796,9 @@ public void failShard(String reason, Throwable e) { } @Override - public Tuple perform(Request request) throws Exception { - Tuple result = shardOperationOnPrimary(request); - result.v2().primaryTerm(indexShard.getPrimaryTerm()); + public PrimaryResult perform(Request request) throws Exception { + PrimaryResult result = shardOperationOnPrimary(request); + result.replicaRequest().primaryTerm(indexShard.getPrimaryTerm()); return result; } @@ -805,20 +850,6 @@ public void onFailure(Throwable shardFailedError) { } } - protected final void processAfterWrite(boolean refresh, IndexShard indexShard, Translog.Location location) { - if (refresh) { - try { - indexShard.refresh("refresh_flag_index"); - } catch (Throwable e) { - // ignore - } - } - if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST && location != null) { - indexShard.sync(location); - } - indexShard.maybeFlush(); - } - /** * Sets the current phase on the task if it isn't null. Pulled into its own * method because its more convenient that way. diff --git a/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java new file mode 100644 index 0000000000000..e50ad7f130634 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/support/replication/TransportWriteAction.java @@ -0,0 +1,227 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.replication; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.support.WriteResponse; +import org.elasticsearch.cluster.action.shard.ShardStateAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.Translog.Location; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; + +import java.util.function.Supplier; + +/** + * Base class for transport actions that modify data in some shard like index, delete, and shardBulk. + */ +public abstract class TransportWriteAction< + Request extends ReplicatedWriteRequest, + Response extends ReplicationResponse & WriteResponse + > extends TransportReplicationAction { + + protected TransportWriteAction(Settings settings, String actionName, TransportService transportService, + ClusterService clusterService, IndicesService indicesService, ThreadPool threadPool, ShardStateAction shardStateAction, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Supplier request, + String executor) { + super(settings, actionName, transportService, clusterService, indicesService, threadPool, shardStateAction, actionFilters, + indexNameExpressionResolver, request, request, executor); + } + + /** + * Called on the primary with a reference to the {@linkplain IndexShard} to modify. + */ + protected abstract WriteResult onPrimaryShard(Request request, IndexShard indexShard) throws Exception; + + /** + * Called once per replica with a reference to the {@linkplain IndexShard} to modify. + * + * @return the translog location of the {@linkplain IndexShard} after the write was completed or null if no write occurred + */ + protected abstract Translog.Location onReplicaShard(Request request, IndexShard indexShard); + + @Override + protected final WritePrimaryResult shardOperationOnPrimary(Request request) throws Exception { + IndexShard indexShard = indexShard(request); + WriteResult result = onPrimaryShard(request, indexShard); + return new WritePrimaryResult(request, result.getResponse(), result.getLocation(), indexShard); + } + + @Override + protected final WriteReplicaResult shardOperationOnReplica(Request request) { + IndexShard indexShard = indexShard(request); + Translog.Location location = onReplicaShard(request, indexShard); + return new WriteReplicaResult(indexShard, request, location); + } + + /** + * Fetch the IndexShard for the request. Protected so it can be mocked in tests. + */ + protected IndexShard indexShard(Request request) { + final ShardId shardId = request.shardId(); + IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + return indexService.getShard(shardId.id()); + } + + /** + * Simple result from a write action. Write actions have static method to return these so they can integrate with bulk. + */ + public static class WriteResult { + private final Response response; + private final Translog.Location location; + + public WriteResult(Response response, @Nullable Location location) { + this.response = response; + this.location = location; + } + + public Response getResponse() { + return response; + } + + public Translog.Location getLocation() { + return location; + } + } + + /** + * Result of taking the action on the primary. + */ + class WritePrimaryResult extends PrimaryResult implements RespondingWriteResult { + boolean finishedAsyncActions; + ActionListener listener = null; + + public WritePrimaryResult(Request request, Response finalResponse, + @Nullable Translog.Location location, + IndexShard indexShard) { + super(request, finalResponse); + /* + * We call this before replication because this might wait for a refresh and that can take a while. This way we wait for the + * refresh in parallel on the primary and on the replica. + */ + postWriteActions(indexShard, request, location, this, logger); + } + + @Override + public synchronized void respond(ActionListener listener) { + this.listener = listener; + respondIfPossible(); + } + + /** + * Respond if the refresh has occurred and the listener is ready. Always called while synchronized on {@code this}. + */ + protected void respondIfPossible() { + if (finishedAsyncActions && listener != null) { + super.respond(listener); + } + } + + @Override + public synchronized void respondAfterAsyncAction(boolean forcedRefresh) { + finalResponse.setForcedRefresh(forcedRefresh); + finishedAsyncActions = true; + respondIfPossible(); + } + } + + /** + * Result of taking the action on the replica. + */ + class WriteReplicaResult extends ReplicaResult implements RespondingWriteResult { + boolean finishedAsyncActions; + private ActionListener listener; + + public WriteReplicaResult(IndexShard indexShard, ReplicatedWriteRequest request, Translog.Location location) { + postWriteActions(indexShard, request, location, this, logger); + } + + @Override + public void respond(ActionListener listener) { + this.listener = listener; + respondIfPossible(); + } + + /** + * Respond if the refresh has occurred and the listener is ready. Always called while synchronized on {@code this}. + */ + protected void respondIfPossible() { + if (finishedAsyncActions && listener != null) { + super.respond(listener); + } + } + + @Override + public synchronized void respondAfterAsyncAction(boolean forcedRefresh) { + finishedAsyncActions = true; + respondIfPossible(); + } + } + + private interface RespondingWriteResult { + void respondAfterAsyncAction(boolean forcedRefresh); + } + + static void postWriteActions(final IndexShard indexShard, + final WriteRequest request, + @Nullable final Translog.Location location, + final RespondingWriteResult respond, + final ESLogger logger) { + boolean pendingOps = false; + boolean immediateRefresh = false; + switch (request.getRefreshPolicy()) { + case IMMEDIATE: + indexShard.refresh("refresh_flag_index"); + immediateRefresh = true; + break; + case WAIT_UNTIL: + if (location != null) { + pendingOps = true; + indexShard.addRefreshListener(location, forcedRefresh -> { + logger.warn("block_until_refresh request ran out of slots and forced a refresh: [{}]", request); + respond.respondAfterAsyncAction(forcedRefresh); + }); + } + break; + case NONE: + break; + } + boolean fsyncTranslog = indexShard.getTranslogDurability() == Translog.Durability.REQUEST && location != null; + if (fsyncTranslog) { + indexShard.sync(location); + } + indexShard.maybeFlush(); + if (pendingOps == false) { + respond.respondAfterAsyncAction(immediateRefresh); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java index 0363ef8fe4312..ca55a63c1d606 100644 --- a/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/action/update/TransportUpdateAction.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; +import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.delete.TransportDeleteAction; import org.elasticsearch.action.index.IndexRequest; @@ -187,6 +188,7 @@ public void onResponse(IndexResponse response) { } else { update.setGetResult(null); } + update.setForcedRefresh(response.forcedRefresh()); listener.onResponse(update); } @@ -219,6 +221,7 @@ protected void doRun() { public void onResponse(IndexResponse response) { UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), response.isCreated()); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), indexSourceBytes)); + update.setForcedRefresh(response.forcedRefresh()); listener.onResponse(update); } @@ -241,11 +244,13 @@ protected void doRun() { }); break; case DELETE: - deleteAction.execute(result.action(), new ActionListener() { + DeleteRequest deleteRequest = result.action(); + deleteAction.execute(deleteRequest, new ActionListener() { @Override public void onResponse(DeleteResponse response) { UpdateResponse update = new UpdateResponse(response.getShardInfo(), response.getShardId(), response.getType(), response.getId(), response.getVersion(), false); update.setGetResult(updateHelper.extractGetResult(request, request.concreteIndex(), response.getVersion(), result.updatedSourceAsMap(), result.updateSourceContentType(), null)); + update.setForcedRefresh(response.forcedRefresh()); listener.onResponse(update); } diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index 9ac77050202ec..0c9c1c67978a6 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -131,7 +131,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult // it has to be a "create!" .create(true) .ttl(ttl) - .refresh(request.refresh()) + .setRefreshPolicy(request.getRefreshPolicy()) .routing(request.routing()) .parent(request.parent()) .consistencyLevel(request.consistencyLevel()); @@ -229,12 +229,13 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult .version(updateVersion).versionType(request.versionType()) .consistencyLevel(request.consistencyLevel()) .timestamp(timestamp).ttl(ttl) - .refresh(request.refresh()); + .setRefreshPolicy(request.getRefreshPolicy()); return new Result(indexRequest, Operation.INDEX, updatedSourceAsMap, updateSourceContentType); } else if ("delete".equals(operation)) { DeleteRequest deleteRequest = Requests.deleteRequest(request.index()).type(request.type()).id(request.id()).routing(routing).parent(parent) .version(updateVersion).versionType(request.versionType()) - .consistencyLevel(request.consistencyLevel()); + .consistencyLevel(request.consistencyLevel()) + .setRefreshPolicy(request.getRefreshPolicy()); return new Result(deleteRequest, Operation.DELETE, updatedSourceAsMap, updateSourceContentType); } else if ("none".equals(operation)) { UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), false); diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 31f219fd4c7e1..e0846c1ce5dcb 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.DocumentRequest; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequest; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseFieldMatcher; @@ -53,7 +54,8 @@ /** */ -public class UpdateRequest extends InstanceShardOperationRequest implements DocumentRequest { +public class UpdateRequest extends InstanceShardOperationRequest + implements DocumentRequest, WriteRequest { private String type; private String id; @@ -72,7 +74,7 @@ public class UpdateRequest extends InstanceShardOperationRequest private VersionType versionType = VersionType.INTERNAL; private int retryOnConflict = 0; - private boolean refresh = false; + private RefreshPolicy refreshPolicy = RefreshPolicy.NONE; private WriteConsistencyLevel consistencyLevel = WriteConsistencyLevel.DEFAULT; @@ -422,18 +424,15 @@ public VersionType versionType() { return this.versionType; } - /** - * Should a refresh be executed post this update operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public UpdateRequest refresh(boolean refresh) { - this.refresh = refresh; + @Override + public UpdateRequest setRefreshPolicy(RefreshPolicy refreshPolicy) { + this.refreshPolicy = refreshPolicy; return this; } - public boolean refresh() { - return this.refresh; + @Override + public RefreshPolicy getRefreshPolicy() { + return refreshPolicy; } public WriteConsistencyLevel consistencyLevel() { @@ -730,7 +729,7 @@ public void readFrom(StreamInput in) throws IOException { script = new Script(in); } retryOnConflict = in.readVInt(); - refresh = in.readBoolean(); + refreshPolicy = RefreshPolicy.readFrom(in); if (in.readBoolean()) { doc = new IndexRequest(); doc.readFrom(in); @@ -767,7 +766,7 @@ public void writeTo(StreamOutput out) throws IOException { script.writeTo(out); } out.writeVInt(retryOnConflict); - out.writeBoolean(refresh); + refreshPolicy.writeTo(out); if (doc == null) { out.writeBoolean(false); } else { diff --git a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java index 30b636f4efc68..403f4265fcdf4 100644 --- a/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/update/UpdateRequestBuilder.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequestBuilder; import org.elasticsearch.action.support.single.instance.InstanceShardOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; @@ -32,9 +33,8 @@ import java.util.Map; -/** - */ -public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder { +public class UpdateRequestBuilder extends InstanceShardOperationRequestBuilder + implements WriteRequestBuilder { public UpdateRequestBuilder(ElasticsearchClient client, UpdateAction action) { super(client, action, new UpdateRequest()); @@ -121,17 +121,6 @@ public UpdateRequestBuilder setVersionType(VersionType versionType) { return this; } - - /** - * Should a refresh be executed post this update operation causing the operation to - * be searchable. Note, heavy indexing should not set this to true. Defaults - * to false. - */ - public UpdateRequestBuilder setRefresh(boolean refresh) { - request.refresh(refresh); - return this; - } - /** * Sets the consistency level of write. Defaults to {@link org.elasticsearch.action.WriteConsistencyLevel#DEFAULT} */ diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 027100b346928..b9d0c6b4c70fa 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -115,6 +115,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.QUERY_STRING_LENIENT_SETTING, IndexSettings.ALLOW_UNMAPPED, IndexSettings.INDEX_CHECK_ON_STARTUP, + IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING, IndexSettings.INDEX_GC_DELETES_SETTING, IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING, diff --git a/core/src/main/java/org/elasticsearch/index/IndexSettings.java b/core/src/main/java/org/elasticsearch/index/IndexSettings.java index 7c8cb4ff8c884..592c1ff112536 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/core/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -115,6 +115,11 @@ public final class IndexSettings { public static final Setting INDEX_GC_DELETES_SETTING = Setting.timeSetting("index.gc_deletes", DEFAULT_GC_DELETES, new TimeValue(-1, TimeUnit.MILLISECONDS), Property.Dynamic, Property.IndexScope); + /** + * The maximum number of refresh listeners allows on this shard. + */ + public static final Setting MAX_REFRESH_LISTENERS_PER_SHARD = Setting.intSetting("index.max_refresh_listeners", 1000, 0, + Property.Dynamic, Property.IndexScope); private final Index index; private final Version version; @@ -145,6 +150,10 @@ public final class IndexSettings { private volatile int maxResultWindow; private volatile int maxRescoreWindow; private volatile boolean TTLPurgeDisabled; + /** + * The maximum number of refresh listeners allows on this shard. + */ + private volatile int maxRefreshListeners; /** * Returns the default search field for this index. @@ -229,6 +238,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti maxResultWindow = scopedSettings.get(MAX_RESULT_WINDOW_SETTING); maxRescoreWindow = scopedSettings.get(MAX_RESCORE_WINDOW_SETTING); TTLPurgeDisabled = scopedSettings.get(INDEX_TTL_DISABLE_PURGE_SETTING); + maxRefreshListeners = scopedSettings.get(MAX_REFRESH_LISTENERS_PER_SHARD); this.mergePolicyConfig = new MergePolicyConfig(logger, this); assert indexNameMatcher.test(indexMetaData.getIndex().getName()); @@ -251,6 +261,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_GC_DELETES_SETTING, this::setGCDeletes); scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING, this::setTranslogFlushThresholdSize); scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval); + scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners); } private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) { @@ -499,6 +510,16 @@ public T getValue(Setting setting) { return scopedSettings.get(setting); } + /** + * The maximum number of refresh listeners allows on this shard. + */ + public int getMaxRefreshListeners() { + return maxRefreshListeners; + } + + private void setMaxRefreshListeners(int maxRefreshListeners) { + this.maxRefreshListeners = maxRefreshListeners; + } IndexScopedSettings getScopedSettings() { return scopedSettings;} } diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index ab142f9dd51a9..87ffb9331a651 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -607,6 +607,7 @@ public final boolean refreshNeeded() { * Synchronously refreshes the engine for new search operations to reflect the latest * changes. */ + @Nullable public abstract void refresh(String source) throws EngineException; /** @@ -999,6 +1000,9 @@ public static class GetResult implements Releasable { public static final GetResult NOT_EXISTS = new GetResult(false, Versions.NOT_FOUND, null); + /** + * Build a realtime get result from the translog. + */ public GetResult(boolean exists, long version, @Nullable Translog.Source source) { this.source = source; this.exists = exists; @@ -1007,6 +1011,9 @@ public GetResult(boolean exists, long version, @Nullable Translog.Source source) this.searcher = null; } + /** + * Build a non-realtime get result from the searcher. + */ public GetResult(Searcher searcher, Versions.DocIdAndVersion docIdAndVersion) { this.exists = true; this.source = null; diff --git a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 8a56feff70f52..13408408e7ec2 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/core/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -25,14 +25,15 @@ import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; import org.apache.lucene.search.similarities.Similarity; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.shard.RefreshListeners; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.store.Store; @@ -40,8 +41,6 @@ import org.elasticsearch.indices.IndexingMemoryController; import org.elasticsearch.threadpool.ThreadPool; -import java.util.function.Function; - /* * Holds all the configuration that is used to create an {@link Engine}. * Once {@link Engine} has been created with this object, changes to this @@ -66,6 +65,8 @@ public final class EngineConfig { private final Engine.EventListener eventListener; private final QueryCache queryCache; private final QueryCachingPolicy queryCachingPolicy; + @Nullable + private final RefreshListeners refreshListeners; /** * Index setting to change the low level lucene codec used for writing new segments. @@ -99,7 +100,7 @@ public EngineConfig(OpenMode openMode, ShardId shardId, ThreadPool threadPool, MergePolicy mergePolicy,Analyzer analyzer, Similarity similarity, CodecService codecService, Engine.EventListener eventListener, TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, - TranslogConfig translogConfig, TimeValue flushMergesAfter) { + TranslogConfig translogConfig, TimeValue flushMergesAfter, RefreshListeners refreshListeners) { if (openMode == null) { throw new IllegalArgumentException("openMode must not be null"); } @@ -125,6 +126,7 @@ public EngineConfig(OpenMode openMode, ShardId shardId, ThreadPool threadPool, this.translogConfig = translogConfig; this.flushMergesAfter = flushMergesAfter; this.openMode = openMode; + this.refreshListeners = refreshListeners; } /** @@ -303,4 +305,10 @@ public enum OpenMode { OPEN_INDEX_AND_TRANSLOG; } + /** + * {@linkplain RefreshListeners} instance to configure. + */ + public RefreshListeners getRefreshListeners() { + return refreshListeners; + } } diff --git a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index c120b07fce892..15667e7942142 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -154,6 +154,10 @@ public InternalEngine(EngineConfig engineConfig) throws EngineException { this.versionMap.setManager(searcherManager); // don't allow commits until we are done with recovering allowCommits.compareAndSet(true, openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG); + if (engineConfig.getRefreshListeners() != null) { + searcherManager.addListener(engineConfig.getRefreshListeners()); + engineConfig.getRefreshListeners().setTranslog(translog); + } success = true; } finally { if (success == false) { diff --git a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java index c30b2e9bf501f..0a55803a5ecb2 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/ShadowEngine.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ReleasableLock; -import org.elasticsearch.index.shard.TranslogRecoveryPerformer; import org.elasticsearch.index.translog.Translog; import java.io.IOException; @@ -68,6 +67,9 @@ public class ShadowEngine extends Engine { public ShadowEngine(EngineConfig engineConfig) { super(engineConfig); + if (engineConfig.getRefreshListeners() != null) { + throw new IllegalArgumentException("ShadowEngine doesn't support RefreshListeners"); + } SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig); final long nonexistentRetryTime = engineConfig.getIndexSettings().getSettings() .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT) diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index adcf36f694f0c..5f950fefb3be5 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -39,7 +39,6 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.RestoreSource; -import org.elasticsearch.client.Client; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Booleans; @@ -132,6 +131,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.BiConsumer; public class IndexShard extends AbstractIndexShardComponent { @@ -204,6 +204,12 @@ public class IndexShard extends AbstractIndexShardComponent { * IndexingMemoryController}). */ private final AtomicBoolean active = new AtomicBoolean(); + /** + * Allows for the registration of listeners that are called when a change becomes visible for search. This is nullable because + * {@linkplain ShadowIndexShard} doesn't support this. + */ + @Nullable + private final RefreshListeners refreshListeners; public IndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardPath path, Store store, IndexCache indexCache, MapperService mapperService, SimilarityService similarityService, IndexFieldDataService indexFieldDataService, @@ -256,6 +262,7 @@ public IndexShard(ShardRouting shardRouting, IndexSettings indexSettings, ShardP suspendableRefContainer = new SuspendableRefContainer(); searcherWrapper = indexSearcherWrapper; primaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); + refreshListeners = buildRefreshListeners(); persistMetadata(shardRouting, null); } @@ -580,6 +587,7 @@ public Engine.GetResult get(Engine.Get get) { */ public void refresh(String source) { verifyNotClosed(); + if (canIndex()) { long bytes = getEngine().getIndexBufferRAMBytesUsed(); writingBytes.addAndGet(bytes); @@ -1531,7 +1539,7 @@ private final EngineConfig newEngineConfig(EngineConfig.OpenMode openMode) { return new EngineConfig(openMode, shardId, threadPool, indexSettings, warmer, store, deletionPolicy, indexSettings.getMergePolicy(), mapperService.indexAnalyzer(), similarityService.similarity(mapperService), codecService, shardEventListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, translogConfig, - IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings())); + IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), refreshListeners); } public Releasable acquirePrimaryOperationLock() { @@ -1627,6 +1635,17 @@ public void onAfter() { return false; } + /** + * Build {@linkplain RefreshListeners} for this shard. Protected so {@linkplain ShadowIndexShard} can override it to return null. + */ + protected RefreshListeners buildRefreshListeners() { + return new RefreshListeners( + indexSettings::getMaxRefreshListeners, + () -> refresh("too_many_listeners"), + threadPool.executor(ThreadPool.Names.LISTENER)::execute, + logger); + } + /** * Simple struct encapsulating a shard failure * @@ -1652,14 +1671,26 @@ EngineFactory getEngineFactory() { } /** - * Returns true iff one or more changes to the engine are not visible to via the current searcher. + * Returns true iff one or more changes to the engine are not visible to via the current searcher *or* there are pending + * refresh listeners. * Otherwise false. * * @throws EngineClosedException if the engine is already closed * @throws AlreadyClosedException if the internal indexwriter in the engine is already closed */ public boolean isRefreshNeeded() { - return getEngine().refreshNeeded(); + return getEngine().refreshNeeded() || (refreshListeners != null && refreshListeners.refreshNeeded()); + } + + /** + * Add a listener for refreshes. + * + * @param location the location to listen for + * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with + * false otherwise. + */ + public void addRefreshListener(Translog.Location location, Consumer listener) { + refreshListeners.addOrNotify(location, listener); } private class IndexShardRecoveryPerformer extends TranslogRecoveryPerformer { diff --git a/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java new file mode 100644 index 0000000000000..ab3e334714af0 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/shard/RefreshListeners.java @@ -0,0 +1,208 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.apache.lucene.search.ReferenceManager; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.index.translog.Translog; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.function.Consumer; +import java.util.function.IntSupplier; + +import static java.util.Objects.requireNonNull; + +/** + * Allows for the registration of listeners that are called when a change becomes visible for search. This functionality is exposed from + * {@link IndexShard} but kept here so it can be tested without standing up the entire thing. + */ +public final class RefreshListeners implements ReferenceManager.RefreshListener { + private final IntSupplier getMaxRefreshListeners; + private final Runnable forceRefresh; + private final Executor listenerExecutor; + private final ESLogger logger; + + /** + * List of refresh listeners. Defaults to null and built on demand because most refresh cycles won't need it. Entries are never removed + * from it, rather, it is nulled and rebuilt when needed again. The (hopefully) rare entries that didn't make the current refresh cycle + * are just added back to the new list. Both the reference and the contents are always modified while synchronized on {@code this}. + */ + private volatile List>> refreshListeners = null; + /** + * The translog location that was last made visible by a refresh. + */ + private volatile Translog.Location lastRefreshedLocation; + + public RefreshListeners(IntSupplier getMaxRefreshListeners, Runnable forceRefresh, Executor listenerExecutor, ESLogger logger) { + this.getMaxRefreshListeners = getMaxRefreshListeners; + this.forceRefresh = forceRefresh; + this.listenerExecutor = listenerExecutor; + this.logger = logger; + } + + /** + * Add a listener for refreshes, calling it immediately if the location is already visible. If this runs out of listener slots then it + * forces a refresh and calls the listener immediately as well. + * + * @param location the location to listen for + * @param listener for the refresh. Called with true if registering the listener ran it out of slots and forced a refresh. Called with + * false otherwise. + */ + public void addOrNotify(Translog.Location location, Consumer listener) { + requireNonNull(listener, "listener cannot be null"); + requireNonNull(location, "location cannot be null"); + + if (lastRefreshedLocation != null && lastRefreshedLocation.compareTo(location) >= 0) { + // Location already visible, just call the listener + listener.accept(false); + return; + } + synchronized (this) { + if (refreshListeners == null) { + refreshListeners = new ArrayList<>(); + } + if (refreshListeners.size() < getMaxRefreshListeners.getAsInt()) { + // We have a free slot so register the listener + refreshListeners.add(new Tuple<>(location, listener)); + return; + } + } + // No free slot so force a refresh and call the listener in this thread + forceRefresh.run(); + listener.accept(true); + } + + /** + * Returns true if there are pending listeners. + */ + public boolean refreshNeeded() { + // No need to synchronize here because we're doing a single volatile read + return refreshListeners != null; + } + + /** + * Setup the translog used to find the last refreshed location. + */ + public void setTranslog(Translog translog) { + this.translog = translog; + } + + // Implementation of ReferenceManager.RefreshListener that adapts Lucene's RefreshListener into Elasticsearch's refresh listeners. + private Translog translog; + /** + * Snapshot of the translog location before the current refresh if there is a refresh going on or null. Doesn't have to be volatile + * because when it is used by the refreshing thread. + */ + private Translog.Location currentRefreshLocation; + + @Override + public void beforeRefresh() throws IOException { + currentRefreshLocation = translog.getLastWriteLocation(); + } + + @Override + public void afterRefresh(boolean didRefresh) throws IOException { + /* + * We intentionally ignore didRefresh here because our timing is a little off. It'd be a useful flag if we knew everything that made + * it into the refresh, but the way we snapshot the translog position before the refresh, things can sneak into the refresh that we + * don't know about. + */ + if (null == currentRefreshLocation) { + /* + * The translog had an empty last write location at the start of the refresh so we can't alert anyone to anything. This + * usually happens during recovery. The next refresh cycle out to pick up this refresh. + */ + return; + } + // First check if we've actually moved forward. If not then just bail immediately. + assert lastRefreshedLocation == null || currentRefreshLocation.compareTo(lastRefreshedLocation) >= 0; + if (lastRefreshedLocation != null && currentRefreshLocation.compareTo(lastRefreshedLocation) == 0) { + return; + } + /* + * Set the lastRefreshedLocation so listeners that come in for locations before that will just execute inline without messing + * around with refreshListeners or synchronizing at all. + */ + lastRefreshedLocation = currentRefreshLocation; + /* + * Grab the current refresh listeners and replace them with null while synchronized. Any listeners that come in after this won't be + * in the list we iterate over and very likely won't be candidates for refresh anyway because we've already moved the + * lastRefreshedLocation. + */ + List>> candidates; + synchronized (this) { + candidates = refreshListeners; + // No listeners to check so just bail early + if (candidates == null) { + return; + } + refreshListeners = null; + } + // Iterate the list of listeners, copying the listeners to fire to one list and those to preserve to another list. + List> listenersToFire = null; + List>> preservedListeners = null; + for (Tuple> tuple : candidates) { + Translog.Location location = tuple.v1(); + Consumer listener = tuple.v2(); + if (location.compareTo(currentRefreshLocation) <= 0) { + if (listenersToFire == null) { + listenersToFire = new ArrayList<>(); + } + listenersToFire.add(listener); + } else { + if (preservedListeners == null) { + preservedListeners = new ArrayList<>(); + } + preservedListeners.add(tuple); + } + } + /* + * Now add any preserved listeners back to the running list of refresh listeners while under lock. We'll try them next time. While + * we were iterating the list of listeners new listeners could have come in. That means that adding all of our preserved listeners + * might push our list of listeners above the maximum number of slots allowed. This seems unlikely because we expect few listeners + * to be preserved. And the next listener while we're full will trigger a refresh anyway. + */ + if (preservedListeners != null) { + synchronized (this) { + if (refreshListeners == null) { + refreshListeners = new ArrayList<>(); + } + refreshListeners.addAll(preservedListeners); + } + } + // Lastly, fire the listeners that are ready on the listener thread pool + if (listenersToFire != null) { + final List> finalListenersToFire = listenersToFire; + listenerExecutor.execute(() -> { + for (Consumer listener : finalListenersToFire) { + try { + listener.accept(false); + } catch (Throwable t) { + logger.warn("Error firing refresh listener", t); + } + } + }); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java index bf35d02fea2c0..e22f684637eb9 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShadowIndexShard.java @@ -31,12 +31,14 @@ import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.threadpool.ThreadPool; import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.function.Consumer; /** * ShadowIndexShard extends {@link IndexShard} to add file synchronization @@ -86,6 +88,12 @@ protected Engine newEngine(EngineConfig config) { return engineFactory.newReadOnlyEngine(config); } + @Override + protected RefreshListeners buildRefreshListeners() { + // ShadowEngine doesn't have a translog so it shouldn't try to support RefreshListeners. + return null; + } + @Override public boolean shouldFlush() { // we don't need to flush since we don't write - all dominated by the primary @@ -96,4 +104,9 @@ public boolean shouldFlush() { public TranslogStats translogStats() { return null; // shadow engine has no translog } + + @Override + public void addRefreshListener(Translog.Location location, Consumer listener) { + throw new UnsupportedOperationException("Can't listen for a refresh on a shadow engine because it doesn't have a translog"); + } } diff --git a/core/src/main/java/org/elasticsearch/index/translog/Translog.java b/core/src/main/java/org/elasticsearch/index/translog/Translog.java index b66c82d4083b2..57847972e4257 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/Translog.java +++ b/core/src/main/java/org/elasticsearch/index/translog/Translog.java @@ -447,6 +447,21 @@ public Location add(Operation operation) throws IOException { } } + /** + * The a {@linkplain Location} that will sort after the {@linkplain Location} returned by the last write but before any locations which + * can be returned by the next write. + */ + public Location getLastWriteLocation() { + try (ReleasableLock lock = readLock.acquire()) { + /* + * We use position = current - 1 and size = Integer.MAX_VALUE here instead of position current and size = 0 for two reasons: + * 1. Translog.Location's compareTo doesn't actually pay attention to size even though it's equals method does. + * 2. It feels more right to return a *position* that is before the next write's position rather than rely on the size. + */ + return new Location(current.generation, current.sizeInBytes() - 1, Integer.MAX_VALUE); + } + } + boolean assertBytesAtLocation(Translog.Location location, BytesReference expectedBytes) throws IOException { // tests can override this ByteBuffer buffer = ByteBuffer.allocate(location.size); diff --git a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index 0109995f80f28..b2c0cc88cf909 100644 --- a/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/core/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.Channels; +import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.shard.ShardId; diff --git a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java index 620418eb08727..d9dbb21e80445 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java @@ -84,7 +84,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, bulkRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); - bulkRequest.refresh(request.paramAsBoolean("refresh", bulkRequest.refresh())); + bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.content(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultPipeline, null, allowExplicitIndex); client.bulk(bulkRequest, new RestBuilderListener(channel) { diff --git a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java index 8e3449344c4a1..29316893504fe 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java @@ -51,7 +51,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, deleteRequest.routing(request.param("routing")); deleteRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing deleteRequest.timeout(request.paramAsTime("timeout", DeleteRequest.DEFAULT_TIMEOUT)); - deleteRequest.refresh(request.paramAsBoolean("refresh", deleteRequest.refresh())); + deleteRequest.setRefreshPolicy(request.param("refresh")); deleteRequest.version(RestActions.parseVersion(request)); deleteRequest.versionType(VersionType.fromString(request.param("version_type"), deleteRequest.versionType())); diff --git a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java index 26dd1eca78d60..f807e68088a2a 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/index/RestIndexAction.java @@ -80,7 +80,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, indexRequest.setPipeline(request.param("pipeline")); indexRequest.source(request.content()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); - indexRequest.refresh(request.paramAsBoolean("refresh", indexRequest.refresh())); + indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); String sOpType = request.param("op_type"); diff --git a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java index 88f90374523be..bdea4e33e6d30 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/update/RestUpdateAction.java @@ -58,7 +58,7 @@ public void handleRequest(final RestRequest request, final RestChannel channel, updateRequest.routing(request.param("routing")); updateRequest.parent(request.param("parent")); updateRequest.timeout(request.paramAsTime("timeout", updateRequest.timeout())); - updateRequest.refresh(request.paramAsBoolean("refresh", updateRequest.refresh())); + updateRequest.setRefreshPolicy(request.param("refresh")); String consistencyLevel = request.param("consistency"); if (consistencyLevel != null) { updateRequest.consistencyLevel(WriteConsistencyLevel.fromString(consistencyLevel)); diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index f2afc1e7f6e75..337f881d41b79 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.Requests; import org.elasticsearch.common.Strings; @@ -180,22 +181,22 @@ public void testSimpleBulk10() throws Exception { public void testBulkRequestWithRefresh() throws Exception { BulkRequest bulkRequest = new BulkRequest(); // We force here a "id is missing" validation error - bulkRequest.add(new DeleteRequest("index", "type", null).refresh(true)); + bulkRequest.add(new DeleteRequest("index", "type", null).setRefreshPolicy(RefreshPolicy.IMMEDIATE)); // We force here a "type is missing" validation error bulkRequest.add(new DeleteRequest("index", null, "id")); - bulkRequest.add(new DeleteRequest("index", "type", "id").refresh(true)); - bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").refresh(true)); - bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").refresh(true)); + bulkRequest.add(new DeleteRequest("index", "type", "id").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new UpdateRequest("index", "type", "id").doc("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); + bulkRequest.add(new IndexRequest("index", "type", "id").source("{}").setRefreshPolicy(RefreshPolicy.IMMEDIATE)); ActionRequestValidationException validate = bulkRequest.validate(); assertThat(validate, notNullValue()); assertThat(validate.validationErrors(), not(empty())); assertThat(validate.validationErrors(), contains( - "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", + "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", "id is missing", "type is missing", - "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", - "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.", - "Refresh is not supported on an item request, set the refresh flag on the BulkRequest instead.")); + "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", + "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.", + "RefreshPolicy is not supported on an item request. Set it on the BulkRequest instead.")); } // issue 15120 diff --git a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java index ff1a24d690014..b26d2531ff0f3 100644 --- a/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java +++ b/core/src/test/java/org/elasticsearch/action/bulk/BulkShardRequestTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESTestCase; @@ -28,9 +29,11 @@ public class BulkShardRequestTests extends ESTestCase { public void testToString() { String index = randomSimpleString(random(), 10); int count = between(1, 100); - BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), false, new BulkItemRequest[count]); + BulkShardRequest r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.NONE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests", r.toString()); - r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), true, new BulkItemRequest[count]); + r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.IMMEDIATE, new BulkItemRequest[count]); assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests and a refresh", r.toString()); + r = new BulkShardRequest(null, new ShardId(index, "ignored", 0), RefreshPolicy.WAIT_UNTIL, new BulkItemRequest[count]); + assertEquals("BulkShardRequest to [" + index + "] containing [" + count + "] requests blocking until refresh", r.toString()); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index 6175f822b6a8a..ebaa5b5c01ea0 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.admin.indices.flush.FlushRequest; diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java index cc7558d1de815..55e2a9d3cf28b 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/ReplicationOperationTests.java @@ -21,16 +21,15 @@ import org.apache.lucene.index.CorruptIndexException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.logging.ESLogger; @@ -102,7 +101,7 @@ public void testReplication() throws Exception { } Request request = new Request(shardId); - PlainActionFuture listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); final ClusterState finalState = state; final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures); final TestReplicationOperation op = new TestReplicationOperation(request, @@ -114,7 +113,7 @@ public void testReplication() throws Exception { assertThat(request.processedOnReplicas, equalTo(expectedReplicas)); assertThat(replicasProxy.failedReplicas, equalTo(expectedFailedShards)); assertTrue("listener is not marked as done", listener.isDone()); - Response.ShardInfo shardInfo = listener.actionGet().getShardInfo(); + ShardInfo shardInfo = listener.actionGet().getShardInfo(); assertThat(shardInfo.getFailed(), equalTo(expectedFailedShards.size())); assertThat(shardInfo.getFailures(), arrayWithSize(expectedFailedShards.size())); assertThat(shardInfo.getSuccessful(), equalTo(1 + expectedReplicas.size() - expectedFailures.size())); @@ -135,7 +134,7 @@ public void testReplicationWithShadowIndex() throws Exception { final ShardRouting primaryShard = indexShardRoutingTable.primaryShard(); Request request = new Request(shardId); - PlainActionFuture listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); final TestReplicationOperation op = new TestReplicationOperation(request, new TestPrimary(primaryShard, primaryTerm), listener, false, false, new TestReplicaProxy(), () -> state, logger, "test"); @@ -143,7 +142,7 @@ public void testReplicationWithShadowIndex() throws Exception { assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true)); assertThat(request.processedOnReplicas, equalTo(Collections.emptySet())); assertTrue("listener is not marked as done", listener.isDone()); - Response.ShardInfo shardInfo = listener.actionGet().getShardInfo(); + ShardInfo shardInfo = listener.actionGet().getShardInfo(); assertThat(shardInfo.getFailed(), equalTo(0)); assertThat(shardInfo.getFailures(), arrayWithSize(0)); assertThat(shardInfo.getSuccessful(), equalTo(1)); @@ -172,7 +171,7 @@ public void testDemotedPrimary() throws Exception { expectedFailures.put(failedReplica, new CorruptIndexException("simulated", (String) null)); Request request = new Request(shardId); - PlainActionFuture listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); final ClusterState finalState = state; final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) { @Override @@ -233,16 +232,16 @@ private void testClusterStateChangeAfterPrimaryOperation(final ShardId shardId, final ShardRouting primaryShard = state.get().routingTable().shardRoutingTable(shardId).primaryShard(); final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm) { @Override - public Tuple perform(Request request) throws Exception { - final Tuple tuple = super.perform(request); + public Result perform(Request request) throws Exception { + Result result = super.perform(request); state.set(changedState); logger.debug("--> state after primary operation:\n{}", state.get().prettyPrint()); - return tuple; + return result; } }; Request request = new Request(shardId); - PlainActionFuture listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, new TestReplicaProxy(), state::get); op.execute(); @@ -296,7 +295,7 @@ public void testWriteConsistency() throws Exception { state.prettyPrint()); final long primaryTerm = state.metaData().index(index).primaryTerm(shardId.id()); final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id()); - PlainActionFuture listener = new PlainActionFuture<>(); + PlainActionFuture listener = new PlainActionFuture<>(); final ShardRouting primaryShard = shardRoutingTable.primaryShard(); final TestReplicationOperation op = new TestReplicationOperation(request, new TestPrimary(primaryShard, primaryTerm), @@ -362,10 +361,7 @@ public void readFrom(StreamInput in) throws IOException { } } - static class Response extends ReplicationResponse { - } - - static class TestPrimary implements ReplicationOperation.Primary { + static class TestPrimary implements ReplicationOperation.Primary { final ShardRouting routing; final long term; @@ -385,12 +381,35 @@ public void failShard(String message, Throwable throwable) { } @Override - public Tuple perform(Request request) throws Exception { + public Result perform(Request request) throws Exception { if (request.processedOnPrimary.compareAndSet(false, true) == false) { fail("processed [" + request + "] twice"); } request.primaryTerm(term); - return new Tuple<>(new Response(), request); + return new Result(request); + } + + static class Result implements ReplicationOperation.PrimaryResult { + private final Request request; + private ShardInfo shardInfo; + + public Result(Request request) { + this.request = request; + } + + @Override + public Request replicaRequest() { + return request; + } + + @Override + public void setShardInfo(ShardInfo shardInfo) { + this.shardInfo = shardInfo; + } + + public ShardInfo getShardInfo() { + return shardInfo; + } } } @@ -436,15 +455,15 @@ public void failShard(ShardRouting replica, ShardRouting primary, String message } } - class TestReplicationOperation extends ReplicationOperation { - public TestReplicationOperation(Request request, Primary primary, ActionListener listener, - Replicas replicas, Supplier clusterStateSupplier) { + class TestReplicationOperation extends ReplicationOperation { + public TestReplicationOperation(Request request, Primary primary, + ActionListener listener, Replicas replicas, Supplier clusterStateSupplier) { this(request, primary, listener, true, false, replicas, clusterStateSupplier, ReplicationOperationTests.this.logger, "test"); } - public TestReplicationOperation(Request request, Primary primary, ActionListener listener, - boolean executeOnReplicas, boolean checkWriteConsistency, Replicas replicas, - Supplier clusterStateSupplier, ESLogger logger, String opType) { + public TestReplicationOperation(Request request, Primary primary, + ActionListener listener, boolean executeOnReplicas, boolean checkWriteConsistency, + Replicas replicas, Supplier clusterStateSupplier, ESLogger logger, String opType) { super(request, primary, listener, executeOnReplicas, checkWriteConsistency, replicas, clusterStateSupplier, logger, opType); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java index 2e81ec712eb5b..afa72ec7526c3 100644 --- a/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportReplicationActionTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.UnavailableShardsException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; @@ -43,7 +42,6 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lease.Releasable; @@ -155,7 +153,7 @@ public void testBlocks() throws ExecutionException, InterruptedException { ClusterBlocks.Builder block = ClusterBlocks.builder() .addGlobalBlock(new ClusterBlock(1, "non retryable", false, true, RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL)); setState(clusterService, ClusterState.builder(clusterService.state()).blocks(block)); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); assertListenerThrows("primary phase should fail operation", listener, ClusterBlockException.class); assertPhase(task, "failed"); @@ -199,7 +197,7 @@ public void testNotStartedPrimary() throws InterruptedException, ExecutionExcept Request request = new Request(shardId).timeout("1ms"); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); assertListenerThrows("unassigned primary didn't cause a timeout", listener, UnavailableShardsException.class); assertPhase(task, "failed"); @@ -245,7 +243,7 @@ public void testNoRerouteOnStaleClusterState() throws InterruptedException, Exec Request request = new Request(shardId).timeout("1ms").routedBasedOnClusterVersion(clusterService.state().version() + 1); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(null, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(null, request, listener); reroutePhase.run(); assertListenerThrows("cluster state too old didn't cause a timeout", listener, UnavailableShardsException.class); @@ -285,7 +283,7 @@ public void testUnknownIndexOrShardOnReroute() throws InterruptedException { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); assertListenerThrows("must throw index not found exception", listener, IndexNotFoundException.class); assertPhase(task, "failed"); @@ -312,7 +310,7 @@ public void testStalePrimaryShardOnReroute() throws InterruptedException { PlainActionFuture listener = new PlainActionFuture<>(); ReplicationTask task = maybeTask(); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear(); assertThat(capturedRequests, arrayWithSize(1)); @@ -364,7 +362,7 @@ public void testRoutePhaseExecutesRequest() { Request request = new Request(shardId); PlainActionFuture listener = new PlainActionFuture<>(); - TransportReplicationAction.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); + Action.ReroutePhase reroutePhase = action.new ReroutePhase(task, request, listener); reroutePhase.run(); assertThat(request.shardId(), equalTo(shardId)); logger.info("--> primary is assigned to [{}], checking request forwarded", primaryNodeId); @@ -393,9 +391,9 @@ public void testPrimaryPhaseExecutesOrDelegatesRequestToRelocationTarget() throw AtomicBoolean executed = new AtomicBoolean(); Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { @Override - protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, - Action.PrimaryShardReference primaryShardReference, - boolean executeOnReplicas) { + protected ReplicationOperation createReplicatedOperation(Request request, + ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, + boolean executeOnReplicas) { return new NoopReplicationOperation(request, actionListener) { public void execute() throws Exception { assertPhase(task, "primary"); @@ -448,9 +446,9 @@ public void testPrimaryPhaseExecutesDelegatedRequestOnRelocationTarget() throws AtomicBoolean executed = new AtomicBoolean(); Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { @Override - protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, - Action.PrimaryShardReference primaryShardReference, - boolean executeOnReplicas) { + protected ReplicationOperation createReplicatedOperation(Request request, + ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, + boolean executeOnReplicas) { return new NoopReplicationOperation(request, actionListener) { public void execute() throws Exception { assertPhase(task, "primary"); @@ -478,9 +476,9 @@ public void testPrimaryReference() throws Exception { }; Action.PrimaryShardReference primary = action.new PrimaryShardReference(shard, releasable); final Request request = new Request(); - Tuple result = primary.perform(request); + Request replicaRequest = primary.perform(request).replicaRequest; - assertThat(result.v2().primaryTerm(), equalTo(primaryTerm)); + assertThat(replicaRequest.primaryTerm(), equalTo(primaryTerm)); final ElasticsearchException exception = new ElasticsearchException("testing"); primary.failShard("test", exception); @@ -582,9 +580,9 @@ public void testShadowIndexDisablesReplication() throws Exception { setState(clusterService, state); Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { @Override - protected ReplicationOperation createReplicatedOperation(Request request, ActionListener actionListener, - Action.PrimaryShardReference primaryShardReference, - boolean executeOnReplicas) { + protected ReplicationOperation createReplicatedOperation(Request request, + ActionListener actionListener, Action.PrimaryShardReference primaryShardReference, + boolean executeOnReplicas) { assertFalse(executeOnReplicas); return new NoopReplicationOperation(request, actionListener); } @@ -608,9 +606,9 @@ public void testCounterOnPrimary() throws Exception { Action.PrimaryOperationTransportHandler primaryPhase = action.new PrimaryOperationTransportHandler() { @Override - protected ReplicationOperation createReplicatedOperation(Request request, ActionListener listener, - Action.PrimaryShardReference primaryShardReference, - boolean executeOnReplicas) { + protected ReplicationOperation createReplicatedOperation(Request request, + ActionListener listener, Action.PrimaryShardReference primaryShardReference, + boolean executeOnReplicas) { assertIndexShardCounter(1); if (throwExceptionOnCreation) { throw new ElasticsearchException("simulated exception, during createReplicatedOperation"); @@ -623,7 +621,7 @@ public void execute() throws Exception { if (throwExceptionOnRun) { throw new ElasticsearchException("simulated exception, during performOnPrimary"); } else if (respondWithError) { - this.finalResponseListener.onFailure(new ElasticsearchException("simulated exception, as a response")); + this.resultListener.onFailure(new ElasticsearchException("simulated exception, as a response")); } else { super.execute(); } @@ -667,13 +665,13 @@ public void testReplicasCounter() throws Exception { final ReplicationTask task = maybeTask(); Action action = new Action(Settings.EMPTY, "testActionWithExceptions", transportService, clusterService, threadPool) { @Override - protected void shardOperationOnReplica(Request request) { + protected ReplicaResult shardOperationOnReplica(Request request) { assertIndexShardCounter(1); assertPhase(task, "replica"); if (throwException) { throw new ElasticsearchException("simulated"); } - super.shardOperationOnReplica(request); + return new ReplicaResult(); } }; final Action.ReplicaOperationTransportHandler replicaOperationTransportHandler = action.new ReplicaOperationTransportHandler(); @@ -765,15 +763,16 @@ protected Response newResponseInstance() { } @Override - protected Tuple shardOperationOnPrimary(Request shardRequest) throws Exception { + protected PrimaryResult shardOperationOnPrimary(Request shardRequest) throws Exception { boolean executedBefore = shardRequest.processedOnPrimary.getAndSet(true); assert executedBefore == false : "request has already been executed on the primary"; - return new Tuple<>(new Response(), shardRequest); + return new PrimaryResult(shardRequest, new Response()); } @Override - protected void shardOperationOnReplica(Request request) { + protected ReplicaResult shardOperationOnReplica(Request request) { request.processedOnReplicas.incrementAndGet(); + return new ReplicaResult(); } @Override @@ -822,15 +821,14 @@ protected Releasable acquireReplicaOperationLock(ShardId shardId, long primaryTe } } - class NoopReplicationOperation extends ReplicationOperation { - - public NoopReplicationOperation(Request request, ActionListener listener) { + class NoopReplicationOperation extends ReplicationOperation { + public NoopReplicationOperation(Request request, ActionListener listener) { super(request, null, listener, true, true, null, null, TransportReplicationActionTests.this.logger, "noop"); } @Override public void execute() throws Exception { - this.finalResponseListener.onResponse(new Response()); + this.resultListener.onResponse(action.new PrimaryResult(null, new Response())); } } diff --git a/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java new file mode 100644 index 0000000000000..7b312959631a7 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/support/replication/TransportWriteActionTests.java @@ -0,0 +1,190 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.support.replication; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.support.WriteResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog; +import org.elasticsearch.index.translog.Translog.Location; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.util.HashSet; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class TransportWriteActionTests extends ESTestCase { + private IndexShard indexShard; + private Translog.Location location; + + @Before + public void initCommonMocks() { + indexShard = mock(IndexShard.class); + location = mock(Translog.Location.class); + } + + public void testPrimaryNoRefreshCall() throws Exception { + noRefreshCall(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond); + } + + public void testReplicaNoRefreshCall() throws Exception { + noRefreshCall(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond); + } + + private void noRefreshCall(ThrowingBiFunction action, + BiConsumer> responder) + throws Exception { + TestRequest request = new TestRequest(); + request.setRefreshPolicy(RefreshPolicy.NONE); // The default, but we'll set it anyway just to be explicit + Result result = action.apply(new TestAction(), request); + CapturingActionListener listener = new CapturingActionListener<>(); + responder.accept(result, listener); + assertNotNull(listener.response); + verify(indexShard, never()).refresh(any()); + verify(indexShard, never()).addRefreshListener(any(), any()); + } + + public void testPrimaryImmediateRefresh() throws Exception { + immediateRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond, r -> assertTrue(r.forcedRefresh)); + } + + public void testReplicaImmediateRefresh() throws Exception { + immediateRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, r -> {}); + } + + private void immediateRefresh(ThrowingBiFunction action, + BiConsumer> responder, + Consumer responseChecker) throws Exception { + TestRequest request = new TestRequest(); + request.setRefreshPolicy(RefreshPolicy.IMMEDIATE); + Result result = action.apply(new TestAction(), request); + CapturingActionListener listener = new CapturingActionListener<>(); + responder.accept(result, listener); + assertNotNull(listener.response); + responseChecker.accept(listener.response); + verify(indexShard).refresh("refresh_flag_index"); + verify(indexShard, never()).addRefreshListener(any(), any()); + } + + public void testPrimaryWaitForRefresh() throws Exception { + waitForRefresh(TestAction::shardOperationOnPrimary, TestAction.WritePrimaryResult::respond, + (r, forcedRefresh) -> assertEquals(forcedRefresh, r.forcedRefresh)); + } + + public void testReplicaWaitForRefresh() throws Exception { + waitForRefresh(TestAction::shardOperationOnReplica, TestAction.WriteReplicaResult::respond, (r, forcedRefresh) -> {}); + } + + private void waitForRefresh(ThrowingBiFunction action, + BiConsumer> responder, + BiConsumer resultChecker) throws Exception { + TestRequest request = new TestRequest(); + request.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + Result result = action.apply(new TestAction(), request); + CapturingActionListener listener = new CapturingActionListener<>(); + responder.accept(result, listener); + assertNull(listener.response); // Haven't reallresponded yet + + @SuppressWarnings({ "unchecked", "rawtypes" }) + ArgumentCaptor> refreshListener = ArgumentCaptor.forClass((Class) Consumer.class); + verify(indexShard, never()).refresh(any()); + verify(indexShard).addRefreshListener(any(), refreshListener.capture()); + + // Now we can fire the listener manually and we'll get a response + boolean forcedRefresh = randomBoolean(); + refreshListener.getValue().accept(forcedRefresh); + assertNotNull(listener.response); + resultChecker.accept(listener.response, forcedRefresh); + } + + private class TestAction extends TransportWriteAction { + protected TestAction() { + super(Settings.EMPTY, "test", mock(TransportService.class), null, null, null, null, new ActionFilters(new HashSet<>()), + new IndexNameExpressionResolver(Settings.EMPTY), TestRequest::new, ThreadPool.Names.SAME); + } + + @Override + protected IndexShard indexShard(TestRequest request) { + return indexShard; + } + + @Override + protected WriteResult onPrimaryShard(TestRequest request, IndexShard indexShard) throws Exception { + return new WriteResult<>(new TestResponse(), location); + } + + @Override + protected Location onReplicaShard(TestRequest request, IndexShard indexShard) { + return location; + } + + @Override + protected TestResponse newResponseInstance() { + return new TestResponse(); + } + } + + private static class TestRequest extends ReplicatedWriteRequest { + public TestRequest() { + setShardId(new ShardId("test", "test", 1)); + } + } + + private static class TestResponse extends ReplicationResponse implements WriteResponse { + boolean forcedRefresh; + + @Override + public void setForcedRefresh(boolean forcedRefresh) { + this.forcedRefresh = forcedRefresh; + } + } + + private static class CapturingActionListener implements ActionListener { + private R response; + + @Override + public void onResponse(R response) { + this.response = response; + } + + @Override + public void onFailure(Throwable e) { + throw new RuntimeException(e); + } + } + + private interface ThrowingBiFunction { + R apply(A a, B b) throws Exception; + } +} diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 08f8970f1e42d..2a16625d03726 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.cluster.metadata.AliasMetaData; @@ -175,10 +176,15 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { assertAcked(admin().indices().prepareAliases().addAlias("test", "tests", termQuery("name", "test"))); logger.info("--> indexing against [test]"); - client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).refresh(true)).actionGet(); - client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).refresh(true)).actionGet(); - client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).refresh(true)).actionGet(); - client().index(indexRequest("test").type("type1").id("4").source(source("4", "something else")).refresh(true)).actionGet(); + client().index(indexRequest("test").type("type1").id("1").source(source("1", "foo test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) + .actionGet(); + client().index(indexRequest("test").type("type1").id("2").source(source("2", "bar test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) + .actionGet(); + client().index(indexRequest("test").type("type1").id("3").source(source("3", "baz test")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) + .actionGet(); + client().index( + indexRequest("test").type("type1").id("4").source(source("4", "something else")).setRefreshPolicy(RefreshPolicy.IMMEDIATE)) + .actionGet(); logger.info("--> checking single filtering alias search"); SearchResponse searchResponse = client().prepareSearch("foos").setQuery(QueryBuilders.matchAllQuery()).get(); diff --git a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java index 0d43580c9bc03..c86535e40c59c 100644 --- a/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteIT.java @@ -22,6 +22,7 @@ import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -225,7 +226,7 @@ private void rerouteWithAllocateLocalGateway(Settings commonSettings) throws Exc assertThat(state.getRoutingNodes().unassigned().size(), equalTo(1)); assertThat(state.getRoutingNodes().node(state.nodes().resolveNode(node_1).getId()).iterator().next().state(), equalTo(ShardRoutingState.STARTED)); - client().prepareIndex("test", "type", "1").setSource("field", "value").setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type", "1").setSource("field", "value").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); final Index index = resolveIndex("test"); logger.info("--> closing all nodes"); diff --git a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java index 4f28cf19d7bae..765cee3b6e8d6 100644 --- a/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java +++ b/core/src/test/java/org/elasticsearch/document/ShardInfoIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.document; -import org.elasticsearch.action.ReplicationResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.bulk.BulkItemResponse; @@ -27,6 +26,7 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.replication.ReplicationResponse; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; diff --git a/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java new file mode 100644 index 0000000000000..b2cb2d9681830 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/WaitUntilRefreshIT.java @@ -0,0 +1,217 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService.ScriptType; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.index.query.QueryBuilders.matchQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoSearchHits; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; + +/** + * Tests that requests with RefreshPolicy.WAIT_UNTIL will be visible when they return. + */ +public class WaitUntilRefreshIT extends ESIntegTestCase { + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(NetworkModule.HTTP_ENABLED.getKey(), true).build(); + } + + @Override + public Settings indexSettings() { + // Use a shorter refresh interval to speed up the tests. We'll be waiting on this interval several times. + return Settings.builder().put(super.indexSettings()).put("index.refresh_interval", "40ms").build(); + } + + @Before + public void createTestIndex() { + createIndex("test"); + } + + public void testIndex() { + IndexResponse index = client().prepareIndex("test", "index", "1").setSource("foo", "bar").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + .get(); + assertEquals(RestStatus.CREATED, index.status()); + assertFalse("request shouldn't have forced a refresh", index.forcedRefresh()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + } + + public void testDelete() throws InterruptedException, ExecutionException { + // Index normally + indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "bar")); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + + // Now delete with blockUntilRefresh + DeleteResponse delete = client().prepareDelete("test", "test", "1").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); + assertTrue("document was deleted", delete.isFound()); + assertFalse("request shouldn't have forced a refresh", delete.forcedRefresh()); + assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get()); + } + + public void testUpdate() throws InterruptedException, ExecutionException { + // Index normally + indexRandom(true, client().prepareIndex("test", "test", "1").setSource("foo", "bar")); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + + // Update with RefreshPolicy.WAIT_UNTIL + UpdateResponse update = client().prepareUpdate("test", "test", "1").setDoc("foo", "baz").setRefreshPolicy(RefreshPolicy.WAIT_UNTIL) + .get(); + assertEquals(2, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); + + // Upsert with RefreshPolicy.WAIT_UNTIL + update = client().prepareUpdate("test", "test", "2").setDocAsUpsert(true).setDoc("foo", "cat") + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); + assertEquals(1, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get(), "2"); + + // Update-becomes-delete with RefreshPolicy.WAIT_UNTIL + update = client().prepareUpdate("test", "test", "2").setScript(new Script("delete_plz", ScriptType.INLINE, "native", emptyMap())) + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).get(); + assertEquals(2, update.getVersion()); + assertFalse("request shouldn't have forced a refresh", update.forcedRefresh()); + assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "cat")).get()); + } + + public void testBulk() { + // Index by bulk with RefreshPolicy.WAIT_UNTIL + BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + bulk.add(client().prepareIndex("test", "test", "1").setSource("foo", "bar")); + assertBulkSuccess(bulk.get()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + + // Update by bulk with RefreshPolicy.WAIT_UNTIL + bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + bulk.add(client().prepareUpdate("test", "test", "1").setDoc("foo", "baz")); + assertBulkSuccess(bulk.get()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1"); + + // Delete by bulk with RefreshPolicy.WAIT_UNTIL + bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + bulk.add(client().prepareDelete("test", "test", "1")); + assertBulkSuccess(bulk.get()); + assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get()); + + // Update makes a noop + bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + bulk.add(client().prepareDelete("test", "test", "1")); + assertBulkSuccess(bulk.get()); + } + + /** + * Tests that an explicit request makes block_until_refresh return. It doesn't check that block_until_refresh doesn't return until the + * explicit refresh if the interval is -1 because we don't have that kind of control over refresh. It can happen all on its own. + */ + public void testNoRefreshInterval() throws InterruptedException, ExecutionException { + client().admin().indices().prepareUpdateSettings("test").setSettings(singletonMap("index.refresh_interval", -1)).get(); + ListenableActionFuture index = client().prepareIndex("test", "index", "1").setSource("foo", "bar") + .setRefreshPolicy(RefreshPolicy.WAIT_UNTIL).execute(); + while (false == index.isDone()) { + client().admin().indices().prepareRefresh("test").get(); + } + assertEquals(RestStatus.CREATED, index.get().status()); + assertFalse("request shouldn't have forced a refresh", index.get().forcedRefresh()); + assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1"); + } + + private void assertBulkSuccess(BulkResponse response) { + assertNoFailures(response); + for (BulkItemResponse item : response) { + assertFalse("request shouldn't have forced a refresh", item.getResponse().forcedRefresh()); + } + } + + @Override + protected Collection> nodePlugins() { + return singleton(DeletePlzPlugin.class); + } + + public static class DeletePlzPlugin extends Plugin { + @Override + public String name() { + return "delete_please"; + } + + @Override + public String description() { + return "adds a script that converts any update into a delete for testing"; + } + + public void onModule(ScriptModule scriptModule) { + scriptModule.registerScript("delete_plz", DeletePlzFactory.class); + } + } + + public static class DeletePlzFactory implements NativeScriptFactory { + @Override + public ExecutableScript newScript(Map params) { + return new ExecutableScript() { + private Map ctx; + + @Override + @SuppressWarnings("unchecked") // Elasicsearch convention + public void setNextVar(String name, Object value) { + if (name.equals("ctx")) { + ctx = (Map) value; + } + } + + @Override + public Object run() { + ctx.put("op", "delete"); + return null; + } + }; + } + + @Override + public boolean needsScores() { + return false; + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 51df3ee03864a..b7da3860003b4 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -170,6 +170,8 @@ public void setUp() throws Exception { .put(IndexSettings.INDEX_GC_DELETES_SETTING, "1h") // make sure this doesn't kick in on us .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), codecName) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD, + between(10, 10 * IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD.get(Settings.EMPTY))) .build()); // TODO randomize more settings threadPool = new ThreadPool(getClass().getName()); store = createStore(); @@ -200,7 +202,7 @@ public EngineConfig copy(EngineConfig config, EngineConfig.OpenMode openMode) { return new EngineConfig(openMode, config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(), config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), config.getQueryCache(), - config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter()); + config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners()); } @Override @@ -291,14 +293,16 @@ public EngineConfig config(IndexSettings indexSettings, Store store, Path transl } catch (IOException e) { throw new ElasticsearchException("can't find index?", e); } - EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings - , null, store, createSnapshotDeletionPolicy(), mergePolicy, - iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), new Engine.EventListener() { + Engine.EventListener listener = new Engine.EventListener() { @Override public void onFailedEngine(String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test } - }, new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); + }; + EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(), + mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), listener, + new TranslogHandler(shardId.getIndexName(), logger), IndexSearcher.getDefaultQueryCache(), + IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), null); return config; } @@ -309,18 +313,20 @@ public void onFailedEngine(String reason, @Nullable Throwable t) { public void testSegments() throws Exception { try (Store store = createStore(); - Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { + Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { List segments = engine.segments(false); assertThat(segments.isEmpty(), equalTo(true)); assertThat(engine.segmentsStats(false).getCount(), equalTo(0L)); assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L)); - // create a doc and refresh + // create two docs and refresh ParsedDocument doc = testParsedDocument("1", "1", "test", null, -1, -1, testDocumentWithTextField(), B_1, null); - engine.index(new Engine.Index(newUid("1"), doc)); - + Engine.Index first = new Engine.Index(newUid("1"), doc); + engine.index(first); ParsedDocument doc2 = testParsedDocument("2", "2", "test", null, -1, -1, testDocumentWithTextField(), B_2, null); - engine.index(new Engine.Index(newUid("2"), doc2)); + Engine.Index second = new Engine.Index(newUid("2"), doc2); + engine.index(second); + assertThat(second.getTranslogLocation(), greaterThan(first.getTranslogLocation())); engine.refresh("test"); segments = engine.segments(false); @@ -2064,10 +2070,11 @@ public void testRecoverFromForeignTranslog() throws IOException { /* create a TranslogConfig that has been created with a different UUID */ TranslogConfig translogConfig = new TranslogConfig(shardId, translog.location(), config.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE); - EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, config.getIndexSettings() - , null, store, createSnapshotDeletionPolicy(), newMergePolicy(), - config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener() - , config.getTranslogRecoveryPerformer(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); + EngineConfig brokenConfig = new EngineConfig(EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, shardId, threadPool, + config.getIndexSettings(), null, store, createSnapshotDeletionPolicy(), newMergePolicy(), config.getAnalyzer(), + config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, + TimeValue.timeValueMinutes(5), config.getRefreshListeners()); try { InternalEngine internalEngine = new InternalEngine(brokenConfig); diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index f71f2d7201956..bed597b7cc5aa 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.UidFieldMapper; +import org.elasticsearch.index.shard.RefreshListeners; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardUtils; @@ -213,7 +214,7 @@ protected InternalEngine createInternalEngine(Store store, Path translogPath) { } protected ShadowEngine createShadowEngine(IndexSettings indexSettings, Store store) { - return new ShadowEngine(config(indexSettings, store, null, null)); + return new ShadowEngine(config(indexSettings, store, null, null, null)); } protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath) { @@ -221,11 +222,12 @@ protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store } protected InternalEngine createInternalEngine(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) { - EngineConfig config = config(indexSettings, store, translogPath, mergePolicy); + EngineConfig config = config(indexSettings, store, translogPath, mergePolicy, null); return new InternalEngine(config); } - public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy) { + public EngineConfig config(IndexSettings indexSettings, Store store, Path translogPath, MergePolicy mergePolicy, + RefreshListeners refreshListeners) { IndexWriterConfig iwc = newIndexWriterConfig(); final EngineConfig.OpenMode openMode; try { @@ -237,14 +239,17 @@ public EngineConfig config(IndexSettings indexSettings, Store store, Path transl } catch (IOException e) { throw new ElasticsearchException("can't find index?", e); } - TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE); - EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings - , null, store, createSnapshotDeletionPolicy(), mergePolicy, - iwc.getAnalyzer(), iwc.getSimilarity() , new CodecService(null, logger), new Engine.EventListener() { + Engine.EventListener eventListener = new Engine.EventListener() { @Override public void onFailedEngine(String reason, @Nullable Throwable t) { // we don't need to notify anybody in this test - }}, null, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5)); + } + }; + TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE); + EngineConfig config = new EngineConfig(openMode, shardId, threadPool, indexSettings, null, store, createSnapshotDeletionPolicy(), + mergePolicy, iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, null, + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, + TimeValue.timeValueMinutes(5), refreshListeners); return config; } @@ -1011,4 +1016,11 @@ public void testDocStats() throws IOException { assertEquals(0, docStats.getDeleted()); primaryEngine.forceMerge(randomBoolean(), 1, false, false, false); } + + public void testRefreshListenersFails() throws IOException { + EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), + new RefreshListeners(null, null, null, logger)); + Exception e = expectThrows(IllegalArgumentException.class, () -> new ShadowEngine(config)); + assertEquals("ShadowEngine doesn't support RefreshListeners", e.getMessage()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java index 702c9b85da427..7b106863341f4 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/AllFieldMapperPositionIncrementGapTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper.all; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.client.Client; import org.elasticsearch.index.query.MatchPhraseQueryBuilder; import org.elasticsearch.plugins.Plugin; @@ -87,7 +88,7 @@ public static void assertGapIsZero(Client client, String indexName, String type) private static void testGap(Client client, String indexName, String type, int positionIncrementGap) throws IOException { client.prepareIndex(indexName, type, "position_gap_test") - .setSource("string1", "one", "string2", "two three").setRefresh(true).get(); + .setSource("string1", "one", "string2", "two three").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); // Baseline - phrase query finds matches in the same field value assertHitCount(client.prepareSearch(indexName) diff --git a/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java new file mode 100644 index 0000000000000..5c83e6805ab28 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -0,0 +1,292 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.shard; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy; +import org.apache.lucene.index.SnapshotDeletionPolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.lucene.uid.Versions; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.engine.InternalEngineTests.TranslogHandler; +import org.elasticsearch.index.fieldvisitor.SingleFieldsVisitor; +import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.translog.TranslogConfig; +import org.elasticsearch.test.DummyShardLock; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.elasticsearch.common.unit.TimeValue.timeValueMillis; + +/** + * Tests how {@linkplain RefreshListeners} interacts with {@linkplain InternalEngine}. + */ +public class RefreshListenersTests extends ESTestCase { + private RefreshListeners listeners; + private Engine engine; + private volatile int maxListeners; + private ThreadPool threadPool; + private Store store; + + @Before + public void setupListeners() throws Exception { + // Setup dependencies of the listeners + maxListeners = randomIntBetween(1, 1000); + listeners = new RefreshListeners( + () -> maxListeners, + () -> engine.refresh("too-many-listeners"), + // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test. + Runnable::run, + logger + ); + + // Now setup the InternalEngine which is much more complicated because we aren't mocking anything + threadPool = new ThreadPool(getTestName()); + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY); + ShardId shardId = new ShardId(new Index("index", "_na_"), 1); + Directory directory = newDirectory(); + DirectoryService directoryService = new DirectoryService(shardId, indexSettings) { + @Override + public Directory newDirectory() throws IOException { + return directory; + } + + @Override + public long throttleTimeInNanos() { + return 0; + } + }; + store = new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId)); + IndexWriterConfig iwc = newIndexWriterConfig(); + TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings, + BigArrays.NON_RECYCLING_INSTANCE); + Engine.EventListener eventListener = new Engine.EventListener() { + @Override + public void onFailedEngine(String reason, @Nullable Throwable t) { + // we don't need to notify anybody in this test + } + }; + EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null, + store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(), + iwc.getSimilarity(), new CodecService(null, logger), eventListener, new TranslogHandler(shardId.getIndexName(), logger), + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, + TimeValue.timeValueMinutes(5), listeners); + engine = new InternalEngine(config); + } + + @After + public void tearDownListeners() throws Exception { + IOUtils.close(engine, store); + terminate(threadPool); + } + + public void testTooMany() throws Exception { + assertFalse(listeners.refreshNeeded()); + Engine.Index index = index("1"); + + // Fill the listener slots + List nonForcedListeners = new ArrayList<>(maxListeners); + for (int i = 0; i < maxListeners; i++) { + DummyRefreshListener listener = new DummyRefreshListener(); + nonForcedListeners.add(listener); + listeners.addOrNotify(index.getTranslogLocation(), listener); + assertTrue(listeners.refreshNeeded()); + } + + // We shouldn't have called any of them + for (DummyRefreshListener listener : nonForcedListeners) { + assertNull("Called listener too early!", listener.forcedRefresh.get()); + } + + // Add one more listener which should cause a refresh. + DummyRefreshListener forcingListener = new DummyRefreshListener(); + listeners.addOrNotify(index.getTranslogLocation(), forcingListener); + assertTrue("Forced listener wasn't forced?", forcingListener.forcedRefresh.get()); + + // That forces all the listeners through. It would be on the listener ThreadPool but we've made all of those execute immediately. + for (DummyRefreshListener listener : nonForcedListeners) { + assertEquals("Expected listener called with unforced refresh!", Boolean.FALSE, listener.forcedRefresh.get()); + } + assertFalse(listeners.refreshNeeded()); + } + + public void testAfterRefresh() throws Exception { + Engine.Index index = index("1"); + engine.refresh("I said so"); + if (randomBoolean()) { + index(randomFrom("1" /* same document */, "2" /* different document */)); + if (randomBoolean()) { + engine.refresh("I said so"); + } + } + + DummyRefreshListener listener = new DummyRefreshListener(); + listeners.addOrNotify(index.getTranslogLocation(), listener); + assertFalse(listener.forcedRefresh.get()); + } + + /** + * Attempts to add a listener at the same time as a refresh occurs by having a background thread force a refresh as fast as it can while + * adding listeners. This can catch the situation where a refresh happens right as the listener is being added such that the listener + * misses the refresh and has to catch the next one. If the listener wasn't able to properly catch the next one then this would fail. + */ + public void testConcurrentRefresh() throws Exception { + AtomicBoolean run = new AtomicBoolean(true); + Thread refresher = new Thread(() -> { + while (run.get()) { + engine.refresh("test"); + } + }); + refresher.start(); + try { + for (int i = 0; i < 100; i++) { + Engine.Index index = index("1"); + + DummyRefreshListener listener = new DummyRefreshListener(); + listeners.addOrNotify(index.getTranslogLocation(), listener); + assertBusy(() -> assertNotNull(listener.forcedRefresh.get())); + assertFalse(listener.forcedRefresh.get()); + } + } finally { + run.set(false); + refresher.join(); + } + } + + /** + * Uses a bunch of threads to index, wait for refresh, and non-realtime get documents to validate that they are visible after waiting + * regardless of what crazy sequence of events causes the refresh listener to fire. + */ + public void testLotsOfThreads() throws Exception { + int threadCount = between(3, 10); + maxListeners = between(1, threadCount * 2); + + // This thread just refreshes every once in a while to cause trouble. + ScheduledFuture refresher = threadPool.scheduleWithFixedDelay(() -> engine.refresh("because test"), timeValueMillis(100)); + + // These threads add and block until the refresh makes the change visible and then do a non-realtime get. + Thread[] indexers = new Thread[threadCount]; + for (int thread = 0; thread < threadCount; thread++) { + final String threadId = String.format(Locale.ROOT, "%04d", thread); + indexers[thread] = new Thread(() -> { + for (int iteration = 1; iteration <= 50; iteration++) { + try { + String testFieldValue = String.format(Locale.ROOT, "%s%04d", threadId, iteration); + Engine.Index index = index(threadId, testFieldValue); + assertEquals(iteration, index.version()); + + DummyRefreshListener listener = new DummyRefreshListener(); + listeners.addOrNotify(index.getTranslogLocation(), listener); + assertBusy(() -> assertNotNull("listener never called", listener.forcedRefresh.get())); + if (threadCount < maxListeners) { + assertFalse(listener.forcedRefresh.get()); + } + + Engine.Get get = new Engine.Get(false, index.uid()); + try (Engine.GetResult getResult = engine.get(get)) { + assertTrue("document not found", getResult.exists()); + assertEquals(iteration, getResult.version()); + SingleFieldsVisitor visitor = new SingleFieldsVisitor("test"); + getResult.docIdAndVersion().context.reader().document(getResult.docIdAndVersion().docId, visitor); + assertEquals(Arrays.asList(testFieldValue), visitor.fields().get("test")); + } + } catch (Throwable t) { + throw new RuntimeException("failure on the [" + iteration + "] iteration of thread [" + threadId + "]", t); + } + } + }); + indexers[thread].start(); + } + + for (Thread indexer: indexers) { + indexer.join(); + } + FutureUtils.cancel(refresher); + } + + private Engine.Index index(String id) { + return index(id, "test"); + } + + private Engine.Index index(String id, String testFieldValue) { + String type = "test"; + String uid = type + ":" + id; + Document document = new Document(); + document.add(new TextField("test", testFieldValue, Field.Store.YES)); + Field uidField = new Field("_uid", type + ":" + id, UidFieldMapper.Defaults.FIELD_TYPE); + Field versionField = new NumericDocValuesField("_version", Versions.MATCH_ANY); + document.add(uidField); + document.add(versionField); + BytesReference source = new BytesArray(new byte[] { 1 }); + ParsedDocument doc = new ParsedDocument(versionField, id, type, null, -1, -1, Arrays.asList(document), source, null); + Engine.Index index = new Engine.Index(new Term("_uid", uid), doc); + engine.index(index); + return index; + } + + private static class DummyRefreshListener implements Consumer { + /** + * When the listener is called this captures it's only argument. + */ + private AtomicReference forcedRefresh = new AtomicReference<>(); + + @Override + public void accept(Boolean forcedRefresh) { + assertNotNull(forcedRefresh); + Boolean oldValue = this.forcedRefresh.getAndSet(forcedRefresh); + assertNull("Listener called twice", oldValue); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java index b4d2423921c0d..889897e21f607 100644 --- a/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java +++ b/core/src/test/java/org/elasticsearch/index/translog/TranslogTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.translog; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.index.Term; import org.apache.lucene.mockfile.FilterFileChannel; @@ -44,6 +45,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.translog.Translog.Location; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.hamcrest.Matchers; @@ -204,18 +206,40 @@ private String randomNonTranslogPatternString(int min, int max) { } public void testRead() throws IOException { + Location loc0 = translog.getLastWriteLocation(); + assertNotNull(loc0); + Translog.Location loc1 = translog.add(new Translog.Index("test", "1", new byte[]{1})); + assertThat(loc1, greaterThan(loc0)); + assertThat(translog.getLastWriteLocation(), greaterThan(loc1)); Translog.Location loc2 = translog.add(new Translog.Index("test", "2", new byte[]{2})); + assertThat(loc2, greaterThan(loc1)); + assertThat(translog.getLastWriteLocation(), greaterThan(loc2)); assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + + Translog.Location lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); + assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); assertThat(translog.read(loc1).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{1}))); assertThat(translog.read(loc2).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{2}))); + Translog.Location loc3 = translog.add(new Translog.Index("test", "2", new byte[]{3})); + assertThat(loc3, greaterThan(loc2)); + assertThat(translog.getLastWriteLocation(), greaterThan(loc3)); assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); + + lastLocBeforeSync = translog.getLastWriteLocation(); translog.sync(); + assertEquals(lastLocBeforeSync, translog.getLastWriteLocation()); assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); translog.prepareCommit(); + /* + * The commit adds to the lastWriteLocation even though is isn't really a write. This is just an implementation artifact but it can + * safely be ignored because the lastWriteLocation continues to be greater than the Location returned from the last write operation + * and less than the location of the next write operation. + */ + assertThat(translog.getLastWriteLocation(), greaterThan(lastLocBeforeSync)); assertThat(translog.read(loc3).getSource().source.toBytesArray(), equalTo(new BytesArray(new byte[]{3}))); translog.commit(); assertNull(translog.read(loc1)); diff --git a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java index 6a4de4b9ff238..6a4d965706a25 100644 --- a/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/AliasRoutingIT.java @@ -21,6 +21,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.test.ESIntegTestCase; @@ -45,7 +46,7 @@ public void testAliasCrudRouting() throws Exception { assertAcked(admin().indices().prepareAliases().addAliasAction(newAddAliasAction("test", "alias0").routing("0"))); logger.info("--> indexing with id [1], and routing [0] using alias"); - client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -72,7 +73,7 @@ public void testAliasCrudRouting() throws Exception { logger.info("--> deleting with no routing, should not delete anything"); - client().prepareDelete("test", "type1", "1").setRefresh(true).execute().actionGet(); + client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); @@ -80,7 +81,7 @@ public void testAliasCrudRouting() throws Exception { } logger.info("--> deleting with routing alias, should delete"); - client().prepareDelete("alias0", "type1", "1").setRefresh(true).execute().actionGet(); + client().prepareDelete("alias0", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); @@ -88,7 +89,7 @@ public void testAliasCrudRouting() throws Exception { } logger.info("--> indexing with id [1], and routing [0] using alias"); - client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -110,7 +111,7 @@ public void testAliasSearchRouting() throws Exception { .addAliasAction(newAddAliasAction("test", "alias01").searchRouting("0,1"))); logger.info("--> indexing with id [1], and routing [0] using alias"); - client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -142,7 +143,7 @@ public void testAliasSearchRouting() throws Exception { } logger.info("--> indexing with id [2], and routing [1] using alias"); - client().prepareIndex("alias1", "type1", "2").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias1", "type1", "2").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -207,7 +208,7 @@ public void testAliasSearchRoutingWithTwoIndices() throws Exception { .addAliasAction(newAddAliasAction("test-b", "alias-ab").searchRouting("1"))); ensureGreen(); // wait for events again to make sure we got the aliases on all nodes logger.info("--> indexing with id [1], and routing [0] using alias to test-a"); - client().prepareIndex("alias-a0", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias-a0", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test-a", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -218,7 +219,7 @@ public void testAliasSearchRoutingWithTwoIndices() throws Exception { } logger.info("--> indexing with id [0], and routing [1] using alias to test-b"); - client().prepareIndex("alias-b1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias-b1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test-a", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -261,9 +262,9 @@ public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue2682() thro .addAliasAction(newAddAliasAction("index", "index_1").routing("1"))); logger.info("--> indexing on index_1 which is an alias for index with routing [1]"); - client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> indexing on index_2 which is a concrete index"); - client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); + client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search all on index_* should find two"); @@ -286,9 +287,9 @@ public void testAliasSearchRoutingWithConcreteAndAliasedIndices_issue3268() thro .addAliasAction(newAddAliasAction("index", "index_1").routing("1"))); logger.info("--> indexing on index_1 which is an alias for index with routing [1]"); - client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("index_1", "type1", "1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> indexing on index_2 which is a concrete index"); - client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefresh(true).execute().actionGet(); + client().prepareIndex("index_2", "type2", "2").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); SearchResponse searchResponse = client().prepareSearch("index_*").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(1).setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(); @@ -307,7 +308,7 @@ public void testIndexingAliasesOverTime() throws Exception { .addAliasAction(newAddAliasAction("test", "alias").routing("3"))); logger.info("--> indexing with id [0], and routing [3]"); - client().prepareIndex("alias", "type1", "0").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias", "type1", "0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); logger.info("--> verifying get and search with routing, should find"); @@ -332,7 +333,7 @@ public void testIndexingAliasesOverTime() throws Exception { .addAliasAction(newAddAliasAction("test", "alias").searchRouting("3,4").indexRouting("4"))); logger.info("--> indexing with id [1], and routing [4]"); - client().prepareIndex("alias", "type1", "1").setSource("field", "value2").setRefresh(true).execute().actionGet(); + client().prepareIndex("alias", "type1", "1").setSource("field", "value2").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should not find anything"); logger.info("--> verifying get and search with routing, should find"); diff --git a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java index 03e6cbf9ef10b..d8cf1e7b5ec61 100644 --- a/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/SimpleRoutingIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.termvectors.MultiTermVectorsResponse; import org.elasticsearch.action.termvectors.TermVectorsRequest; import org.elasticsearch.action.termvectors.TermVectorsResponse; @@ -56,7 +57,8 @@ public void testSimpleCrudRouting() throws Exception { ensureGreen(); logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -67,21 +69,22 @@ public void testSimpleCrudRouting() throws Exception { } logger.info("--> deleting with no routing, should not delete anything"); - client().prepareDelete("test", "type1", "1").setRefresh(true).execute().actionGet(); + client().prepareDelete("test", "type1", "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); } logger.info("--> deleting with routing, should delete"); - client().prepareDelete("test", "type1", "1").setRouting("0").setRefresh(true).execute().actionGet(); + client().prepareDelete("test", "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); assertThat(client().prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(false)); } logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -97,7 +100,8 @@ public void testSimpleSearchRouting() { ensureGreen(); logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type1", "1").setRouting("0").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); logger.info("--> verifying get with no routing, should not find anything"); for (int i = 0; i < 5; i++) { assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(false)); @@ -125,7 +129,7 @@ public void testSimpleSearchRouting() { } logger.info("--> indexing with id [2], and routing [1]"); - client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex("test", "type1", "2").setRouting("1").setSource("field", "value1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> search with no routing, should fine two"); for (int i = 0; i < 5; i++) { @@ -165,12 +169,13 @@ public void testRequiredRoutingCrudApis() throws Exception { ensureGreen(); logger.info("--> indexing with id [1], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with no routing, should fail"); logger.info("--> indexing with id [1], with no routing, should fail"); try { - client().prepareIndex(indexOrAlias(), "type1", "1").setSource("field", "value1").setRefresh(true).execute().actionGet(); + client().prepareIndex(indexOrAlias(), "type1", "1").setSource("field", "value1").get(); fail("index with missing routing when routing is required should fail"); } catch (ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); @@ -183,7 +188,7 @@ public void testRequiredRoutingCrudApis() throws Exception { logger.info("--> deleting with no routing, should fail"); try { - client().prepareDelete(indexOrAlias(), "type1", "1").setRefresh(true).execute().actionGet(); + client().prepareDelete(indexOrAlias(), "type1", "1").get(); fail("delete with missing routing when routing is required should fail"); } catch (ElasticsearchException e) { assertThat(e.unwrapCause(), instanceOf(RoutingMissingException.class)); @@ -223,7 +228,7 @@ public void testRequiredRoutingCrudApis() throws Exception { assertThat(getResponse.getSourceAsMap().get("field"), equalTo("value2")); } - client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefresh(true).execute().actionGet(); + client().prepareDelete(indexOrAlias(), "type1", "1").setRouting("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); for (int i = 0; i < 5; i++) { try { @@ -320,7 +325,8 @@ public void testRequiredRoutingMappingVariousAPIs() throws Exception { logger.info("--> indexing with id [1], and routing [0]"); client().prepareIndex(indexOrAlias(), "type1", "1").setRouting("0").setSource("field", "value1").get(); logger.info("--> indexing with id [2], and routing [0]"); - client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2").setRefresh(true).get(); + client().prepareIndex(indexOrAlias(), "type1", "2").setRouting("0").setSource("field", "value2") + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); logger.info("--> verifying get with id [1] with routing [0], should succeed"); assertThat(client().prepareGet(indexOrAlias(), "type1", "1").setRouting("0").execute().actionGet().isExists(), equalTo(true)); diff --git a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java index 90f5c65c066ca..f8ca1e1aaf7b2 100644 --- a/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java +++ b/core/src/test/java/org/elasticsearch/search/child/ChildQuerySearchIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery; import org.elasticsearch.common.settings.Settings; @@ -754,10 +755,11 @@ public void testParentChildQueriesCanHandleNoRelevantTypesInIndex() throws Excep assertNoFailures(response); assertThat(response.getHits().totalHits(), equalTo(0L)); - client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject()).setRefresh(true) - .get(); + client().prepareIndex("test", "child1").setSource(jsonBuilder().startObject().field("text", "value").endObject()) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); - response = client().prepareSearch("test").setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get(); + response = client().prepareSearch("test") + .setQuery(QueryBuilders.hasChildQuery("child", matchQuery("text", "value"), ScoreMode.None)).get(); assertNoFailures(response); assertThat(response.getHits().totalHits(), equalTo(0L)); diff --git a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java index 812e43918d9d2..484edf61f7b3a 100644 --- a/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java +++ b/core/src/test/java/org/elasticsearch/search/suggest/CompletionSuggestSearch2xIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -473,7 +474,7 @@ public void testThatUpgradeToMultiFieldTypeWorks() throws Exception { .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id)) .addMapping(TYPE, mapping)); client().prepareIndex(INDEX, TYPE, "1") - .setRefresh(true) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); @@ -496,7 +497,7 @@ public void testThatUpgradeToMultiFieldTypeWorks() throws Exception { ).execute().actionGet(); assertSuggestions(suggestResponse, "suggs"); - client().prepareIndex(INDEX, TYPE, "1").setRefresh(true) + client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); @@ -522,7 +523,7 @@ public void testThatUpgradeToMultiFieldsWorks() throws Exception { .addMapping(TYPE, mapping) .setSettings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, PRE2X_VERSION.id))); client().prepareIndex(INDEX, TYPE, "1") - .setRefresh(true) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); @@ -545,7 +546,7 @@ public void testThatUpgradeToMultiFieldsWorks() throws Exception { ).execute().actionGet(); assertSuggestions(suggestResponse, "suggs"); - client().prepareIndex(INDEX, TYPE, "1").setRefresh(true) + client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").endObject()).get(); ensureGreen(INDEX); @@ -731,10 +732,10 @@ public void testThatStatsAreWorking() throws Exception { assertThat(putMappingResponse.isAcknowledged(), is(true)); // Index two entities - client().prepareIndex(INDEX, TYPE, "1").setRefresh(true) + client().prepareIndex(INDEX, TYPE, "1").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Foo Fighters").field(otherField, "WHATEVER").endObject()) .get(); - client().prepareIndex(INDEX, TYPE, "2").setRefresh(true) + client().prepareIndex(INDEX, TYPE, "2").setRefreshPolicy(RefreshPolicy.IMMEDIATE) .setSource(jsonBuilder().startObject().field(FIELD, "Bar Fighters").field(otherField, "WHATEVER2").endObject()) .get(); @@ -1040,7 +1041,7 @@ public void testMaxFieldLength() throws IOException { .startArray("input").value(str).endArray() .field("output", "foobar") .endObject().endObject() - ).setRefresh(true).get(); + ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); // need to flush and refresh, because we keep changing the same document // we have to make sure that segments without any live documents are deleted flushAndRefresh(); @@ -1074,7 +1075,7 @@ public void testVeryLongInput() throws IOException { .startArray("input").value(longString).endArray() .field("output", "foobar") .endObject().endObject() - ).setRefresh(true).get(); + ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); } @@ -1096,7 +1097,7 @@ public void testReservedChars() throws IOException { .startArray("input").value(string).endArray() .field("output", "foobar") .endObject().endObject() - ).setRefresh(true).get(); + ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); fail("expected MapperParsingException"); } catch (MapperParsingException expected) {} } @@ -1116,7 +1117,7 @@ public void testIssue5930() throws IOException { .startObject() .field(FIELD, string) .endObject() - ).setRefresh(true).get(); + ).setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); try { client().prepareSearch(INDEX).addAggregation(AggregationBuilders.terms("suggest_agg").field(FIELD) @@ -1148,11 +1149,11 @@ public void testIndexingUnrelatedNullValue() throws Exception { ensureGreen(); client().prepareIndex(INDEX, TYPE, "1").setSource(FIELD, "strings make me happy", FIELD + "_1", "nulls make me sad") - .setRefresh(true).get(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); try { client().prepareIndex(INDEX, TYPE, "2").setSource(FIELD, null, FIELD + "_1", "nulls make me sad") - .setRefresh(true).get(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); fail("Expected MapperParsingException for null value"); } catch (MapperParsingException e) { // make sure that the exception has the name of the field causing the error diff --git a/docs/reference/docs.asciidoc b/docs/reference/docs.asciidoc index f3b30e7f0c347..04049663e848f 100644 --- a/docs/reference/docs.asciidoc +++ b/docs/reference/docs.asciidoc @@ -15,6 +15,8 @@ This section describes the following CRUD APIs: .Multi-document APIs * <> * <> +* <> +* <> NOTE: All CRUD APIs are single-index APIs. The `index` parameter accepts a single index name, or an `alias` which points to a single index. @@ -42,3 +44,5 @@ include::docs/reindex.asciidoc[] include::docs/termvectors.asciidoc[] include::docs/multi-termvectors.asciidoc[] + +include::docs/refresh.asciidoc[] diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index ee8f75c6cddb5..6b0a5b2e40df4 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -167,12 +167,8 @@ are the same). [[bulk-refresh]] === Refresh -The `refresh` parameter can be set to `true` in order to refresh the relevant -primary and replica shards immediately after the bulk operation has occurred -and make it searchable, instead of waiting for the normal refresh interval to -expire. Setting it to `true` can trigger additional load, and may slow down -indexing. Due to its costly nature, the `refresh` parameter is set on the bulk request level -and is not supported on each individual bulk item. +Control when the changes made by this request are visible to search. See +<>. [float] [[bulk-update]] diff --git a/docs/reference/docs/delete.asciidoc b/docs/reference/docs/delete.asciidoc index 175c07d005eb4..18a370fc416d4 100644 --- a/docs/reference/docs/delete.asciidoc +++ b/docs/reference/docs/delete.asciidoc @@ -113,11 +113,9 @@ is the same). [[delete-refresh]] === Refresh -The `refresh` parameter can be set to `true` in order to refresh the relevant -primary and replica shards after the delete operation has occurred and make it -searchable. Setting it to `true` should be done after careful thought and -verification that this does not cause a heavy load on the system (and slows -down indexing). +Control when the changes made by this request are visible to search. See +<>. + [float] [[delete-timeout]] diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index eb1e45d616027..aa62b65292e5c 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -30,7 +30,8 @@ The result of the above index operation is: "_type" : "tweet", "_id" : "1", "_version" : 1, - "created" : true + "created" : true, + "forced_refresh": false } -------------------------------------------------- // TESTRESPONSE[s/"successful" : 2/"successful" : 1/] @@ -221,7 +222,8 @@ The result of the above index operation is: "_type" : "tweet", "_id" : "6a8ca01c-7896-48e9-81cc-9f70661fcb32", "_version" : 1, - "created" : true + "created" : true, + "forced_refresh": false } -------------------------------------------------- // TESTRESPONSE[s/6a8ca01c-7896-48e9-81cc-9f70661fcb32/$body._id/ s/"successful" : 2/"successful" : 1/] @@ -385,13 +387,8 @@ replication group have indexed the document (sync replication). [[index-refresh]] === Refresh -To refresh the shard (not the whole index) immediately after the operation -occurs, so that the document appears in search results immediately, the -`refresh` parameter can be set to `true`. Setting this option to `true` should -*ONLY* be done after careful thought and verification that it does not lead to -poor performance, both from an indexing and a search standpoint. Note, getting -a document using the get API is completely realtime and doesn't require a -refresh. +Control when the changes made by this request are visible to search. See +<>. [float] [[index-noop]] diff --git a/docs/reference/docs/refresh.asciidoc b/docs/reference/docs/refresh.asciidoc new file mode 100644 index 0000000000000..3e5153341c8ae --- /dev/null +++ b/docs/reference/docs/refresh.asciidoc @@ -0,0 +1,109 @@ +[[docs-refresh]] +== `?refresh` + +The <>, <>, <>, and +<> APIs support setting `refresh` to control when changes made +by this request are made visible to search. These are the allowed values: + +Empty string or `true`:: + +Refresh the relevant primary and replica shards (not the whole index) +immediately after the operation occurs, so that the updated document appears +in search results immediately. This should *ONLY* be done after careful thought +and verification that it does not lead to poor performance, both from an +indexing and a search standpoint. + +`wait_for`:: + +Wait for the changes made by the request to be made visible by a refresh before +replying. This doesn't force an immediate refresh, rather, it waits for a +refresh happen. Elasticsearch automatically refreshes shards that have changed +every `index.refresh_interval` which defaults to one second. That setting is +<>. The <> API will also +cause the request to return, as will setting `refresh` to `true` on any of the +APIs that support it. + +`false` (the default):: + +Take no refresh related actions. The changes made by this request will be made +visible at some point after the request returns. + +=== Choosing which setting to use + +Unless you have a good reason to wait for the change to become visible always +use `refresh=false`, or, because that is the default, just leave the `refresh` +parameter out of the URL. That is the simplest and fastest choice. + +If you absolutely must have the changes made by a request visible synchronously +with the request then you must get to pick between putting more load on +Elasticsearch (`true`) and waiting longer for the response (`wait_for`). Here +are a few points that should inform that decision: + +* The more changes being made to the index the more work `wait_for` saves +compared to `true`. In the case that the index is only changed once every +`index.refresh_interval` then it saves no work. +* `true` creates less efficient indexes constructs (tiny segments) that must +later be merged into more efficient index constructs (larger segments). Meaning +that the cost of `true` is payed at index time to create the tiny segment, at +search time to search the tiny segment, and at merge time to make the larger +segments. +* Never start multiple `refresh=wait_for` requests in a row. Instead batch them +into a single bulk request with `refresh=wait_for` and Elasticsearch will start +them all in parallel and return only when they have all finished. +* If the refresh interval is set to `-1`, disabling the automatic refreshes, +then requests with `refresh=wait_for` will wait indefinitely until some action +causes a refresh. Conversely, setting `index.refresh_interval` to something +shorter than the default like `200ms` will make `refresh=wait_for` come back +faster, but it'll still generate inefficient segments. +* `refresh=wait_for` only affects the request that it is on, but, by forcing a +refresh immediately, `refresh=true` will affect other ongoing request. In +general, if you have a running system you don't wish to disturb then +`refresh=wait_for` is a smaller modification. + +=== `refresh=wait_for` Can Force a Refresh + +If a `refresh=wait_for` request comes in when there are already +`index.max_refresh_listeners` (defaults to 1000) requests waiting for a refresh +on that shard then that request will behave just as though it had `refresh` set +to `true` instead: it will force a refresh. This keeps the promise that when a +`refresh=wait_for` request returns that its changes are visible for search +while preventing unchecked resource usage for blocked requests. If a request +forced a refresh because it ran out of listener slots then its response will +contain `"forced_refresh": true`. + +Bulk requests only take up one slot on each shard that they touch no matter how +many times they modify the shard. + +=== Examples + +These will create a document and immediately refresh the index so it is visible: + +[source,js] +-------------------------------------------------- +PUT /test/test/1?refresh +{"test": "test"} +PUT /test/test/2?refresh=true +{"test": "test"} +-------------------------------------------------- +// CONSOLE + +These will create a document without doing anything to make it visible for +search: + +[source,js] +-------------------------------------------------- +PUT /test/test/3 +{"test": "test"} +PUT /test/test/4?refresh=true +{"test": "test"} +-------------------------------------------------- +// CONSOLE + +This will create a document and wait for it to become visible for search: + +[source,js] +-------------------------------------------------- +PUT /test/test/4?refresh=wait_for +{"test": "test"} +-------------------------------------------------- +// CONSOLE diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 405f9b0494bf0..39261c5d21f22 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -235,9 +235,8 @@ The write consistency of the index/delete operation. `refresh`:: -Refresh the relevant primary and replica shards (not the whole index) -immediately after the operation occurs, so that the updated document appears -in search results immediately. +Control when the changes made by this request are visible to search. See +<>. `fields`:: diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc index c7119af416884..3cc95e5af9a3e 100644 --- a/docs/reference/index-modules.asciidoc +++ b/docs/reference/index-modules.asciidoc @@ -136,6 +136,11 @@ specific index module: experimental[] Disables the purge of <> on the current index. +`index.max_refresh_listeners`:: + + Maximum number of refresh listeners available on each shard of the index. + These listeners are used to implement <>. + [float] === Settings in other index modules diff --git a/docs/reference/migration/migrate_5_0/docs.asciidoc b/docs/reference/migration/migrate_5_0/docs.asciidoc index 85e4e901e5cb7..9149eed6142b5 100644 --- a/docs/reference/migration/migrate_5_0/docs.asciidoc +++ b/docs/reference/migration/migrate_5_0/docs.asciidoc @@ -1,6 +1,16 @@ [[breaking_50_document_api_changes]] === Document API changes +==== `?refresh` no longer supports truthy and falsy values +The `?refresh` request parameter used to accept any value other than `false`, +`0`, `off`, and `no` to mean "make the changes from this request visible for +search immediately." Now it only accepts `?refresh` and `?refresh=true` to +mean that. You can set it to `?refresh=false` and the request will take no +refresh-related action. The same is true if you leave `refresh` off of the +url entirely. If you add `?refresh=wait_for` Elasticsearch will wait for the +changes to become visible before replying to the request but won't take any +immediate refresh related action. See <>. + ==== Reindex and Update By Query Before 5.0.0 `_reindex` and `_update_by_query` only retried bulk failures so they used the following response format: diff --git a/docs/reference/migration/migrate_5_0/java.asciidoc b/docs/reference/migration/migrate_5_0/java.asciidoc index da97d360b43d8..a16f2a9ff9b50 100644 --- a/docs/reference/migration/migrate_5_0/java.asciidoc +++ b/docs/reference/migration/migrate_5_0/java.asciidoc @@ -304,3 +304,8 @@ The `setQuery(BytesReference)` method have been removed in favor of using `setQu Removed the `getMemoryAvailable` method from `OsStats`, which could be previously accessed calling `clusterStatsResponse.getNodesStats().getOs().getMemoryAvailable()`. + +=== setRefresh(boolean) has been deprecated + +`setRefresh(boolean)` has been deprecated in favor of `setRefreshPolicy(RefreshPolicy)` because there +are now three options. It will be removed in 5.0. diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java index dd9daa75e7665..13868566eac17 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/BulkTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; @@ -584,7 +585,7 @@ public void testThatFailedUpdateRequestReturnsCorrectType() throws Exception { .add(new IndexRequest("test", "type", "4").source("{ \"title\" : \"Great Title of doc 4\" }")) .add(new IndexRequest("test", "type", "5").source("{ \"title\" : \"Great Title of doc 5\" }")) .add(new IndexRequest("test", "type", "6").source("{ \"title\" : \"Great Title of doc 6\" }")) - .setRefresh(true) + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .get(); assertNoFailures(indexBulkItemResponse); @@ -622,7 +623,7 @@ public void testThatMissingIndexDoesNotAbortFullBulkRequest() throws Exception{ .add(new IndexRequest("bulkindex2", "index2_type").source("text", "hallo2")) .add(new UpdateRequest("bulkindex2", "index2_type", "2").doc("foo", "bar")) .add(new DeleteRequest("bulkindex2", "index2_type", "3")) - .refresh(true); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE); client().bulk(bulkRequest).get(); SearchResponse searchResponse = client().prepareSearch("bulkindex*").get(); @@ -643,10 +644,10 @@ public void testFailedRequestsOnClosedIndex() throws Exception { client().prepareIndex("bulkindex1", "index1_type", "1").setSource("text", "test").get(); assertAcked(client().admin().indices().prepareClose("bulkindex1")); - BulkRequest bulkRequest = new BulkRequest(); + BulkRequest bulkRequest = new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE); bulkRequest.add(new IndexRequest("bulkindex1", "index1_type", "1").source("text", "hallo1")) .add(new UpdateRequest("bulkindex1", "index1_type", "1").doc("foo", "bar")) - .add(new DeleteRequest("bulkindex1", "index1_type", "1")).refresh(true); + .add(new DeleteRequest("bulkindex1", "index1_type", "1")); BulkResponse bulkResponse = client().bulk(bulkRequest).get(); assertThat(bulkResponse.hasFailures(), is(true)); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java index ed9b65130a90f..20c8c7f8b5877 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorIT.java @@ -26,6 +26,7 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; import org.elasticsearch.client.Requests; import org.elasticsearch.common.geo.builders.ShapeBuilders; import org.elasticsearch.common.lucene.search.function.CombineFunction; @@ -294,8 +295,8 @@ public void storePercolateQueriesOnRecreatedIndex() throws Exception { .field("color", "blue") .field("query", termQuery("field1", "value1")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); cluster().wipeIndices("test"); createIndex("test"); @@ -308,8 +309,8 @@ public void storePercolateQueriesOnRecreatedIndex() throws Exception { .field("color", "blue") .field("query", termQuery("field1", "value1")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); } // see #2814 @@ -338,8 +339,8 @@ public void testPercolateCustomAnalyzer() throws Exception { .field("source", "productizer") .field("query", QueryBuilders.constantScoreQuery(QueryBuilders.queryStringQuery("filingcategory:s"))) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); refresh(); PercolateResponse percolate = preparePercolate(client()) @@ -417,8 +418,8 @@ public void testMultiplePercolators() throws Exception { .field("color", "blue") .field("query", termQuery("field1", "value1")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); logger.info("--> register a query 2"); client().prepareIndex(INDEX_NAME, TYPE_NAME, "bubu") @@ -426,8 +427,8 @@ public void testMultiplePercolators() throws Exception { .field("color", "green") .field("query", termQuery("field1", "value2")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); PercolateResponse percolate = preparePercolate(client()) .setIndices(INDEX_NAME).setDocumentType("type1") @@ -461,8 +462,8 @@ public void testDynamicAddingRemovingQueries() throws Exception { .field("color", "blue") .field("query", termQuery("field1", "value1")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); PercolateResponse percolate = preparePercolate(client()) .setIndices(INDEX_NAME).setDocumentType("type1") @@ -478,8 +479,8 @@ public void testDynamicAddingRemovingQueries() throws Exception { .field("color", "green") .field("query", termQuery("field1", "value2")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); percolate = preparePercolate(client()) .setIndices(INDEX_NAME).setDocumentType("type1") @@ -495,8 +496,8 @@ public void testDynamicAddingRemovingQueries() throws Exception { .field("color", "red") .field("query", termQuery("field1", "value2")) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); PercolateSourceBuilder sourceBuilder = new PercolateSourceBuilder() .setDoc(docBuilder().setDoc(jsonBuilder().startObject().field("field1", "value2").endObject())) @@ -510,7 +511,7 @@ public void testDynamicAddingRemovingQueries() throws Exception { assertThat(convertFromTextArray(percolate.getMatches(), INDEX_NAME), arrayContaining("susu")); logger.info("--> deleting query 1"); - client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefresh(true).execute().actionGet(); + client().prepareDelete(INDEX_NAME, TYPE_NAME, "kuku").setRefreshPolicy(RefreshPolicy.IMMEDIATE).get(); percolate = preparePercolate(client()) .setIndices(INDEX_NAME).setDocumentType("type1") @@ -1461,8 +1462,8 @@ public void testPercolateNonMatchingConstantScoreQuery() throws Exception { .must(QueryBuilders.queryStringQuery("root")) .must(QueryBuilders.termQuery("message", "tree")))) .endObject()) - .setRefresh(true) - .execute().actionGet(); + .setRefreshPolicy(RefreshPolicy.IMMEDIATE) + .get(); refresh(); PercolateResponse percolate = preparePercolate(client()) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json index 590054b04a425..a75daf35204e3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json @@ -22,8 +22,9 @@ "description" : "Explicit write consistency setting for the operation" }, "refresh": { - "type" : "boolean", - "description" : "Refresh the index after performing the operation" + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes." }, "routing": { "type" : "string", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json index be09c0179d4be..5bb0e3fed4c50 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json @@ -33,8 +33,9 @@ "description" : "ID of parent document" }, "refresh": { - "type" : "boolean", - "description" : "Refresh the index after performing the operation" + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes." }, "routing": { "type" : "string", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json index 5c13f67c2121d..b7f7eeb9ef531 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json @@ -38,8 +38,9 @@ "description" : "ID of the parent document" }, "refresh": { - "type" : "boolean", - "description" : "Refresh the index after performing the operation" + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes." }, "routing": { "type" : "string", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json index 20fc3524283e5..4a3f134301d89 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json @@ -41,8 +41,9 @@ "description": "ID of the parent document. Is is only used for routing and when for the upsert request" }, "refresh": { - "type": "boolean", - "description": "Refresh the index after performing the operation" + "type" : "enum", + "options": ["true", "false", "wait_for"], + "description" : "If `true` then refresh the effected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` (the default) then do nothing with refreshes." }, "retry_on_conflict": { "type": "number", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml new file mode 100644 index 0000000000000..4906975bfab15 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/50_refresh.yaml @@ -0,0 +1,48 @@ +--- +"refresh=true immediately makes changes are visible in search": + - do: + bulk: + refresh: true + body: | + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}} + {"f1": "v1", "f2": 42} + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}} + {"f1": "v2", "f2": 47} + + - do: + count: + index: test_index + - match: {count: 2} + +--- +"refresh=empty string immediately makes changes are visible in search": + - do: + bulk: + refresh: "" + body: | + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}} + {"f1": "v1", "f2": 42} + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}} + {"f1": "v2", "f2": 47} + + - do: + count: + index: test_index + - match: {count: 2} + + +--- +"refresh=wait_for waits until changes are visible in search": + - do: + bulk: + refresh: wait_for + body: | + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}} + {"f1": "v1", "f2": 42} + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}} + {"f1": "v2", "f2": 47} + + - do: + count: + index: test_index + - match: {count: 2} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml index 99bfbc3cff62e..90dc28bcfc083 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/60_refresh.yaml @@ -33,8 +33,9 @@ index: test_1 type: test id: 2 - refresh: 1 + refresh: true body: { foo: bar } + - is_true: forced_refresh - do: search: @@ -44,3 +45,42 @@ query: { term: { _id: 2 }} - match: { hits.total: 1 } + +--- +"When refresh url parameter is an empty string that means \"refresh immediately\"": + - do: + create: + index: test_1 + type: test + id: 1 + refresh: "" + body: { foo: bar } + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + + - match: { hits.total: 1 } + +--- +"refresh=wait_for waits until changes are visible in search": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + refresh: wait_for + - is_false: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml index 4d3f9fe039dd7..9ea6bc033decd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/50_refresh.yaml @@ -19,7 +19,7 @@ type: test id: 1 body: { foo: bar } - refresh: 1 + refresh: true # If you wonder why this document get 3 as an id instead of 2, it is because the # current routing algorithm would route 1 and 2 to the same shard while we need @@ -30,7 +30,8 @@ type: test id: 3 body: { foo: bar } - refresh: 1 + refresh: true + - is_true: forced_refresh - do: search: @@ -61,7 +62,7 @@ index: test_1 type: test id: 3 - refresh: 1 + refresh: true # If a replica shard where doc 1 is located gets initialized at this point, doc 1 # won't be found by the following search as the shard gets automatically refreshed @@ -75,3 +76,72 @@ query: { terms: { _id: [1,3] }} - match: { hits.total: 1 } + +--- +"When refresh url parameter is an empty string that means \"refresh immediately\"": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + refresh: true + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 1 } + + - do: + delete: + index: test_1 + type: test + id: 1 + refresh: "" + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 0 } + +--- +"refresh=wait_for waits until changes are visible in search": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + refresh: true + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 1 } + + - do: + delete: + index: test_1 + type: test + id: 1 + refresh: wait_for + - is_false: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 0 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml index af6ea59766fc0..4ee2641143279 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/60_refresh.yaml @@ -33,8 +33,9 @@ index: test_1 type: test id: 2 - refresh: 1 + refresh: true body: { foo: bar } + - is_true: forced_refresh - do: search: @@ -44,3 +45,42 @@ query: { term: { _id: 2 }} - match: { hits.total: 1 } + +--- +"When refresh url parameter is an empty string that means \"refresh immediately\"": + - do: + index: + index: test_1 + type: test + id: 1 + refresh: "" + body: { foo: bar } + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + + - match: { hits.total: 1 } + +--- +"refresh=wait_for waits until changes are visible in search": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + refresh: wait_for + - is_false: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml index 6048292ceabff..8c0e7e66c9740 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/60_refresh.yaml @@ -35,10 +35,11 @@ index: test_1 type: test id: 2 - refresh: 1 + refresh: true body: doc: { foo: baz } upsert: { foo: bar } + - is_true: forced_refresh - do: search: @@ -48,3 +49,70 @@ query: { term: { _id: 2 }} - match: { hits.total: 1 } + +--- +"When refresh url parameter is an empty string that means \"refresh immediately\"": + - do: + index: + index: test_1 + type: test + id: 1 + refresh: true + body: { foo: bar } + - is_true: forced_refresh + + - do: + update: + index: test_1 + type: test + id: 1 + refresh: "" + body: + doc: {cat: dog} + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { cat: dog }} + + - match: { hits.total: 1 } + +--- +"refresh=wait_for waits until changes are visible in search": + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + refresh: true + - is_true: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { term: { _id: 1 }} + - match: { hits.total: 1 } + + - do: + update: + index: test_1 + type: test + id: 1 + refresh: wait_for + body: + doc: { test: asdf } + - is_false: forced_refresh + + - do: + search: + index: test_1 + type: test + body: + query: { match: { test: asdf } } + - match: { hits.total: 1 }