From 9ff4a95ee4ce3982508965f780afb4343053ae1a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 13:09:08 +0300 Subject: [PATCH 01/96] allow to filter also by node _name and _id, make sure to reroute properly after cluster update settings --- .../TransportClusterUpdateSettingsAction.java | 32 +++++-- .../cluster/node/DiscoveryNodeFilters.java | 19 +++- .../allocation/FilteringAllocationTests.java | 94 +++++++++++++++++++ 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/FilteringAllocationTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 0a805352ac373..7a4800cdaa0a9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -73,7 +74,7 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe final AtomicReference failureRef = new AtomicReference(); final CountDownLatch latch = new CountDownLatch(1); - clusterService.submitStateUpdateTask("cluster_update_settings", new ClusterStateUpdateTask() { + clusterService.submitStateUpdateTask("cluster_update_settings", new ProcessedClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { try { boolean changed = false; @@ -108,17 +109,30 @@ public class TransportClusterUpdateSettingsAction extends TransportMasterNodeOpe .transientSettings(transientSettings.build()); - ClusterState updatedState = ClusterState.builder().state(currentState).metaData(metaData).build(); - - // now, reroute in case things change that require it (like number of replicas) - RoutingAllocation.Result routingResult = allocationService.reroute(updatedState); - updatedState = newClusterStateBuilder().state(updatedState).routingResult(routingResult).build(); - - return updatedState; - } finally { + return ClusterState.builder().state(currentState).metaData(metaData).build(); + } catch (Exception e) { latch.countDown(); + logger.warn("failed to update cluster settings", e); + return currentState; + } finally { + // we don't release the latch here, only after we rerouted } } + + @Override public void clusterStateProcessed(ClusterState clusterState) { + // now, reroute + clusterService.submitStateUpdateTask("reroute_after_cluster_update_settings", new ClusterStateUpdateTask() { + @Override public ClusterState execute(ClusterState currentState) { + try { + // now, reroute in case things change that require it (like number of replicas) + RoutingAllocation.Result routingResult = allocationService.reroute(currentState); + return newClusterStateBuilder().state(currentState).routingResult(routingResult).build(); + } finally { + latch.countDown(); + } + } + }); + } }); try { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java index c41022271fb44..f1e11d0f71ee8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeFilters.java @@ -68,10 +68,25 @@ public boolean match(DiscoveryNode node) { } InetSocketTransportAddress inetAddress = (InetSocketTransportAddress) node.address(); for (String value : values) { - if (!Regex.simpleMatch(value, inetAddress.address().getAddress().getHostAddress())) { - return false; + if (Regex.simpleMatch(value, inetAddress.address().getAddress().getHostAddress())) { + return true; + } + } + return false; + } else if ("_id".equals(attr)) { + for (String value : values) { + if (node.id().equals(value)) { + return true; } } + return false; + } else if ("_name".equals(attr)) { + for (String value : values) { + if (Regex.simpleMatch(value, node.name())) { + return true; + } + } + return false; } else { String nodeAttributeValue = node.attributes().get(attr); if (nodeAttributeValue == null) { diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/FilteringAllocationTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/FilteringAllocationTests.java new file mode 100644 index 0000000000000..1efc14aa966ce --- /dev/null +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/FilteringAllocationTests.java @@ -0,0 +1,94 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.cluster.allocation; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; + +import static org.elasticsearch.common.settings.ImmutableSettings.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + */ +public class FilteringAllocationTests extends AbstractNodesTests { + + private final ESLogger logger = Loggers.getLogger(FilteringAllocationTests.class); + + @AfterMethod public void cleanAndCloseNodes() throws Exception { + closeAllNodes(); + } + + @Test public void testDecommissionNodeNoReplicas() throws Exception { + logger.info("--> starting 2 nodes"); + startNode("node1"); + startNode("node2"); + + logger.info("--> creating an index with no replicas"); + client("node1").admin().indices().prepareCreate("test") + .setSettings(settingsBuilder().put("index.number_of_replicas", 0)) + .execute().actionGet(); + + ClusterHealthResponse clusterHealthResponse = client("node1").admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + assertThat(clusterHealthResponse.timedOut(), equalTo(false)); + + logger.info("--> index some data"); + for (int i = 0; i < 100; i++) { + client("node1").prepareIndex("test", "type", Integer.toString(i)).setSource("field", "value" + i).execute().actionGet(); + } + client("node1").admin().indices().prepareRefresh().execute().actionGet(); + assertThat(client("node1").prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().count(), equalTo(100l)); + + logger.info("--> decommission the second node"); + client("node1").admin().cluster().prepareUpdateSettings() + .setTransientSettings(settingsBuilder().put("cluster.routing.allocation.exclude._name", "node2")) + .execute().actionGet(); + + Thread.sleep(200); + + clusterHealthResponse = client("node1").admin().cluster().prepareHealth() + .setWaitForGreenStatus() + .setWaitForRelocatingShards(0) + .execute().actionGet(); + assertThat(clusterHealthResponse.timedOut(), equalTo(false)); + + logger.info("--> verify all are allocated on node1 now"); + ClusterState clusterState = client("node1").admin().cluster().prepareState().execute().actionGet().state(); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + for (ShardRouting shardRouting : indexShardRoutingTable) { + assertThat(clusterState.nodes().get(shardRouting.currentNodeId()).name(), equalTo("node1")); + } + } + } + + client("node1").admin().indices().prepareRefresh().execute().actionGet(); + assertThat(client("node1").prepareCount().setQuery(QueryBuilders.matchAllQuery()).execute().actionGet().count(), equalTo(100l)); + } +} From 84ae001731dac124546714a1a90893da6d420db0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 13:31:40 +0300 Subject: [PATCH 02/96] if ttl field mappers don't exists (yet), ignore it for that shard --- .../java/org/elasticsearch/indices/ttl/IndicesTTLService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java index 5e54b2a63ef02..05f296c584cd4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/ttl/IndicesTTLService.java @@ -135,6 +135,9 @@ private List getShardsToPurge() { for (IndexService indexService : indicesService) { // should be optimized with the hasTTL flag FieldMappers ttlFieldMappers = indexService.mapperService().name(TTLFieldMapper.NAME); + if (ttlFieldMappers == null) { + continue; + } // check if ttl is enabled for at least one type of this index boolean hasTTLEnabled = false; for (FieldMapper ttlFieldMapper : ttlFieldMappers) { From bf70836e9251d2a7f92e1d328f8f9fab3a0501d7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 14:06:54 +0300 Subject: [PATCH 03/96] don't do double int[] lookup for ordinal --- .../index/field/data/bytes/MultiValueByteFieldData.java | 2 +- .../index/field/data/doubles/MultiValueDoubleFieldData.java | 2 +- .../index/field/data/floats/MultiValueFloatFieldData.java | 2 +- .../index/field/data/ints/MultiValueIntFieldData.java | 2 +- .../index/field/data/longs/MultiValueLongFieldData.java | 2 +- .../index/field/data/shorts/MultiValueShortFieldData.java | 2 +- .../index/field/data/strings/MultiValueStringFieldData.java | 2 +- .../index/mapper/geo/MultiValueGeoPointFieldData.java | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java index e68b1cd5ad8ad..553b82a238b35 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/bytes/MultiValueByteFieldData.java @@ -160,7 +160,7 @@ public MultiValueByteFieldData(String fieldName, int[][] ordinals, byte[] values int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java index 92426e3667eee..2f6290d850ccf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/doubles/MultiValueDoubleFieldData.java @@ -149,7 +149,7 @@ public MultiValueDoubleFieldData(String fieldName, int[][] ordinals, double[] va int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java index a1ed1c15a9aad..6ca2cdf9719ae 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/floats/MultiValueFloatFieldData.java @@ -160,7 +160,7 @@ public MultiValueFloatFieldData(String fieldName, int[][] ordinals, float[] valu int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java index dad25b498fb4e..38d89ad641d08 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/ints/MultiValueIntFieldData.java @@ -160,7 +160,7 @@ public MultiValueIntFieldData(String fieldName, int[][] ordinals, int[] values) int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java index 4b36631373db6..e52adfcfb7521 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/longs/MultiValueLongFieldData.java @@ -162,7 +162,7 @@ public MultiValueLongFieldData(String fieldName, int[][] ordinals, long[] values int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java index 322243aa3deb5..39907bf18f467 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/shorts/MultiValueShortFieldData.java @@ -160,7 +160,7 @@ public MultiValueShortFieldData(String fieldName, int[][] ordinals, short[] valu int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java index a0264c9aef966..d5c6f944c15fd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/field/data/strings/MultiValueStringFieldData.java @@ -90,7 +90,7 @@ public MultiValueStringFieldData(String fieldName, int[][] ordinals, String[] va int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java index 3e9b49e88cc51..9e02dedfee57f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/MultiValueGeoPointFieldData.java @@ -123,7 +123,7 @@ public MultiValueGeoPointFieldData(String fieldName, int[][] ordinals, double[] int loc = ordinal[docId]; if (loc != 0) { found = true; - proc.onOrdinal(docId, ordinal[docId]); + proc.onOrdinal(docId, loc); } } if (!found) { From 052f9aac1f9a002c67a17b884481994c86127108 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 15:21:22 +0300 Subject: [PATCH 04/96] allow to force flush and use it when optimizing --- .../admin/indices/flush/FlushRequest.java | 19 +++++++++++++++++++ .../indices/flush/ShardFlushRequest.java | 9 ++++++++- .../indices/flush/TransportFlushAction.java | 2 +- .../elasticsearch/index/engine/Engine.java | 12 +++++++++++- .../index/engine/robin/RobinEngine.java | 8 ++++---- .../admin/indices/flush/RestFlushAction.java | 8 +++++++- 6 files changed, 50 insertions(+), 8 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java index 87e44b2f04c47..7f4da8bfa84df 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/FlushRequest.java @@ -42,6 +42,8 @@ public class FlushRequest extends BroadcastOperationRequest { private boolean refresh = false; + private boolean force = false; + private boolean full = false; FlushRequest() { @@ -88,6 +90,21 @@ public FlushRequest full(boolean full) { return this; } + /** + * Force flushing, even if one is possibly not needed. + */ + public boolean force() { + return force; + } + + /** + * Force flushing, even if one is possibly not needed. + */ + public FlushRequest force(boolean force) { + this.force = force; + return this; + } + /** * Should the listener be called on a separate thread if needed. */ @@ -108,11 +125,13 @@ public FlushRequest full(boolean full) { super.writeTo(out); out.writeBoolean(refresh); out.writeBoolean(full); + out.writeBoolean(force); } @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); refresh = in.readBoolean(); full = in.readBoolean(); + force = in.readBoolean(); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java index e82828ce4768b..3104d72ad0b10 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/ShardFlushRequest.java @@ -31,8 +31,8 @@ class ShardFlushRequest extends BroadcastShardOperationRequest { private boolean refresh; - private boolean full; + private boolean force; ShardFlushRequest() { } @@ -41,6 +41,7 @@ public ShardFlushRequest(String index, int shardId, FlushRequest request) { super(index, shardId); this.refresh = request.refresh(); this.full = request.full(); + this.force = request.force(); } public boolean refresh() { @@ -51,15 +52,21 @@ public boolean full() { return this.full; } + public boolean force() { + return this.force; + } + @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); refresh = in.readBoolean(); full = in.readBoolean(); + force = in.readBoolean(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeBoolean(refresh); out.writeBoolean(full); + out.writeBoolean(force); } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java index b38e5a6bcb741..d85ad209f3b5d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/flush/TransportFlushAction.java @@ -111,7 +111,7 @@ public class TransportFlushAction extends TransportBroadcastOperationActionfalse. @@ -216,8 +217,17 @@ public Flush full(boolean full) { return this; } + public boolean force() { + return this.force; + } + + public Flush force(boolean force) { + this.force = force; + return this; + } + @Override public String toString() { - return "full[" + full + "], refresh[" + refresh + "]"; + return "full[" + full + "], refresh[" + refresh + "], force[" + force + "]"; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 390715f4e699e..fd34ab6782547 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -798,7 +798,7 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { indexWriter.close(false); indexWriter = createWriter(); - if (flushNeeded) { + if (flushNeeded || flush.force()) { flushNeeded = false; long translogId = translogIdGenerator.incrementAndGet(); indexWriter.commit(MapBuilder.newMapBuilder().put(Translog.TRANSLOG_ID_KEY, Long.toString(translogId)).map()); @@ -827,7 +827,7 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { throw new FlushNotAllowedEngineException(shardId, "Recovery is in progress, flush is not allowed"); } - if (flushNeeded) { + if (flushNeeded || flush.force()) { flushNeeded = false; try { long translogId = translogIdGenerator.incrementAndGet(); @@ -913,7 +913,7 @@ private void refreshVersioningTable(long time) { @Override public void optimize(Optimize optimize) throws EngineException { if (optimize.flush()) { - flush(new Flush()); + flush(new Flush().force(true)); } if (optimizeMutex.compareAndSet(false, true)) { rwl.readLock().lock(); @@ -950,7 +950,7 @@ private void refreshVersioningTable(long time) { indexWriter.waitForMerges(); } if (optimize.flush()) { - flush(new Flush()); + flush(new Flush().force(true)); } if (optimize.refresh()) { refresh(new Refresh(false).force(true)); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java index 0330a5dc8136a..0ac69dc42b3b6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/flush/RestFlushAction.java @@ -27,7 +27,12 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestXContentBuilder; @@ -63,6 +68,7 @@ public class RestFlushAction extends BaseRestHandler { flushRequest.operationThreading(operationThreading); flushRequest.refresh(request.paramAsBoolean("refresh", flushRequest.refresh())); flushRequest.full(request.paramAsBoolean("full", flushRequest.full())); + flushRequest.force(request.paramAsBoolean("force", flushRequest.force())); client.admin().indices().flush(flushRequest, new ActionListener() { @Override public void onResponse(FlushResponse response) { try { From 4279016f307f56674097f422ba0da001c491dc2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Dev=C3=A8ze?= Date: Fri, 9 Sep 2011 12:22:55 +0200 Subject: [PATCH 05/96] add default ttl value support --- .../xcontent/support/XContentMapValues.java | 16 +++++++++ .../index/mapper/internal/TTLFieldMapper.java | 34 ++++++++++++++++--- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index 9a03ce0424449..e565104972bb6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -19,6 +19,8 @@ package org.elasticsearch.common.xcontent.support; +import org.elasticsearch.common.unit.TimeValue; + import java.util.List; import java.util.Map; @@ -146,4 +148,18 @@ public static boolean nodeBooleanValue(Object node) { String value = node.toString(); return !(value.equals("false") || value.equals("0") || value.equals("off")); } + + public static TimeValue nodeTimeValue(Object node, TimeValue defaultValue) { + if (node == null) { + return defaultValue; + } + return nodeTimeValue(node); + } + + public static TimeValue nodeTimeValue(Object node) { + if (node instanceof Number) { + return TimeValue.timeValueMillis(((Number) node).longValue()); + } + return TimeValue.parseTimeValue(node.toString(), null); + } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index d6669d51e5605..ed9955dabc2cd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -54,11 +54,13 @@ public static class Defaults extends LongFieldMapper.Defaults { public static final Field.Store STORE = Field.Store.YES; public static final Field.Index INDEX = Field.Index.NOT_ANALYZED; public static final boolean ENABLED = false; + public static final long DEFAULT = -1; } public static class Builder extends NumberFieldMapper.Builder { private boolean enabled = Defaults.ENABLED; + private long defaultTTL = Defaults.DEFAULT; public Builder() { super(Defaults.NAME); @@ -71,8 +73,13 @@ public Builder enabled(boolean enabled) { return builder; } + public Builder defaultTTL(long defaultTTL) { + this.defaultTTL = defaultTTL; + return builder; + } + @Override public TTLFieldMapper build(BuilderContext context) { - return new TTLFieldMapper(store, index, enabled); + return new TTLFieldMapper(store, index, enabled, defaultTTL); } } @@ -85,6 +92,11 @@ public static class TypeParser implements Mapper.TypeParser { Object fieldNode = entry.getValue(); if (fieldName.equals("enabled")) { builder.enabled(nodeBooleanValue(fieldNode)); + } else if (fieldName.equals("default")) { + TimeValue ttlTimeValue = nodeTimeValue(fieldNode, null); + if (ttlTimeValue != null) { + builder.defaultTTL(ttlTimeValue.millis()); + } } } return builder; @@ -92,22 +104,28 @@ public static class TypeParser implements Mapper.TypeParser { } private boolean enabled; + private long defaultTTL; public TTLFieldMapper() { - this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED); + this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.DEFAULT); } - protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled) { + protected TTLFieldMapper(Field.Store store, Field.Index index, boolean enabled, long defaultTTL) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE); this.enabled = enabled; + this.defaultTTL = defaultTTL; } public boolean enabled() { return this.enabled; } + public long defaultTTL() { + return this.defaultTTL; + } + // Overrides valueForSearch to display live value of remaining ttl @Override public Object valueForSearch(Fieldable field) { long now; @@ -157,9 +175,12 @@ public Object valueForSearch(long expirationTime) { @Override protected Fieldable parseCreateField(ParseContext context) throws IOException, AlreadyExpiredException { if (enabled) { - long timestamp = context.sourceToParse().timestamp(); long ttl = context.sourceToParse().ttl(); + if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value + ttl = defaultTTL; + } if (ttl > 0) { // a ttl has been provided either externally or in the _source + long timestamp = context.sourceToParse().timestamp(); long expire = new Date(timestamp + ttl).getTime(); long now = System.currentTimeMillis(); // there is not point indexing already expired doc @@ -175,13 +196,16 @@ public Object valueForSearch(long expirationTime) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // if all are defaults, no sense to write it at all - if (enabled == Defaults.ENABLED) { + if (enabled == Defaults.ENABLED && defaultTTL == Defaults.DEFAULT) { return builder; } builder.startObject(CONTENT_TYPE); if (enabled != Defaults.ENABLED) { builder.field("enabled", enabled); } + if (defaultTTL != Defaults.DEFAULT) { + builder.field("default", defaultTTL); + } builder.endObject(); return builder; } From a6eaf8c2c323fcca88a733e4a837c29da46a6a12 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 16:57:30 +0300 Subject: [PATCH 06/96] upgrade to aws sdk 1.2.7 --- .idea/modules/plugin-cloud-aws.iml | 2 +- plugins/cloud/aws/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/modules/plugin-cloud-aws.iml b/.idea/modules/plugin-cloud-aws.iml index abc574da936ce..d93a058dbe100 100644 --- a/.idea/modules/plugin-cloud-aws.iml +++ b/.idea/modules/plugin-cloud-aws.iml @@ -18,7 +18,7 @@ - + diff --git a/plugins/cloud/aws/build.gradle b/plugins/cloud/aws/build.gradle index 4439c791202cd..153f6f4718909 100644 --- a/plugins/cloud/aws/build.gradle +++ b/plugins/cloud/aws/build.gradle @@ -32,13 +32,13 @@ configurations { dependencies { compile project(':elasticsearch') - compile("com.amazonaws:aws-java-sdk:1.2.4") { transitive = false } + compile("com.amazonaws:aws-java-sdk:1.2.7") { transitive = false } runtime("commons-logging:commons-logging:1.1.1") { transitive = false } runtime("commons-codec:commons-codec:1.3") { transitive = false } runtime("org.apache.httpcomponents:httpclient:4.1.1") { transitive = false } runtime("org.apache.httpcomponents:httpcore:4.1") { transitive = false } - distLib("com.amazonaws:aws-java-sdk:1.2.4") { transitive = false } + distLib("com.amazonaws:aws-java-sdk:1.2.7") { transitive = false } distLib("commons-codec:commons-codec:1.3") { transitive = false } distLib("commons-logging:commons-logging:1.1.1") { transitive = false } distLib("org.apache.httpcomponents:httpclient:4.1.1") { transitive = false } From 5610d90c2d479fba03b3def2fc7357942d2be72d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 9 Sep 2011 17:25:25 +0300 Subject: [PATCH 07/96] add custom config for plugins path as well --- config/elasticsearch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 2b7ecc8cd8334..d4c093bff6216 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -6,6 +6,7 @@ #path.data: /path/to/data #path.work: /path/to/work #path.logs: /path/to/logs +#path.plugins: /path/to/plugins # Force all memory to be locked, forcing the JVM to never swap #bootstrap.mlockall: true From 8532dc84e949938536f42f427eb551bbcb0e7cd3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 11 Sep 2011 00:53:39 +0300 Subject: [PATCH 08/96] Query with stopwords executed directly against a type fails, closes #1320. --- .../org/elasticsearch/index/query/support/QueryParsers.java | 3 +++ .../org/elasticsearch/search/SearchContextException.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java index bb61ec3b43f18..9af1cffce208d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/support/QueryParsers.java @@ -77,6 +77,9 @@ public static MultiTermQuery.RewriteMethod parseRewriteMethod(@Nullable String r public static Query wrapSmartNameQuery(Query query, @Nullable MapperService.SmartNameFieldMappers smartFieldMappers, QueryParseContext parseContext) { + if (query == null) { + return null; + } if (smartFieldMappers == null) { return query; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchContextException.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchContextException.java index 6299472d6afbe..c0f679835d149 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchContextException.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/SearchContextException.java @@ -38,7 +38,11 @@ private static String buildMessage(SearchContext context, String msg) { StringBuilder sb = new StringBuilder(); sb.append('[').append(context.shardTarget().index()).append("][").append(context.shardTarget().shardId()).append("]: "); if (context.parsedQuery() != null) { - sb.append("query[").append(context.parsedQuery().query()).append("],"); + try { + sb.append("query[").append(context.parsedQuery().query()).append("],"); + } catch (Exception e) { + sb.append("query[_failed_to_string_],"); + } } sb.append("from[").append(context.from()).append("],size[").append(context.size()).append("]"); if (context.sort() != null) { From e6f277474e06333d2a34f14f89341a0b5282ccc7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 11 Sep 2011 01:06:57 +0300 Subject: [PATCH 09/96] Thread Pool: Blocking thread pool type configuration fails, closes #1321. --- config/elasticsearch.yml | 14 ++++++++++++++ .../org/elasticsearch/threadpool/ThreadPool.java | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index d4c093bff6216..7bb231cf4c58c 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -38,3 +38,17 @@ # Unicast Discovery (disable multicast) #discovery.zen.ping.multicast.enabled: false #discovery.zen.ping.unicast.hosts: ["host1", "host2"] + +threadpool: + search: + type: blocking + min: 5 + max: 20 + wait_time: 30s + keep_alive: 90s + index: + type: blocking + min: 3 + max: 10 + wait_time: 30s + keep_alive: 90s diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 4ba9ec724df0e..93de2b647c078 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -195,9 +195,9 @@ private Executor build(String name, String defaultType, @Nullable Settings setti TimeValue keepAlive = settings.getAsTime("keep_alive", defaultSettings.getAsTime("keep_alive", timeValueMinutes(5))); int min = settings.getAsInt("min", defaultSettings.getAsInt("min", 1)); int size = settings.getAsInt("size", defaultSettings.getAsInt("size", Runtime.getRuntime().availableProcessors() * 5)); - SizeValue capacity = settings.getAsSize("capacity", defaultSettings.getAsSize("capacity", new SizeValue(0))); + SizeValue capacity = settings.getAsSize("queue_size", defaultSettings.getAsSize("queue_size", new SizeValue(1000))); TimeValue waitTime = settings.getAsTime("wait_time", defaultSettings.getAsTime("wait_time", timeValueSeconds(60))); - logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], keep_alive [{}], wait_time [{}]", name, type, min, size, keepAlive, waitTime); + logger.debug("creating thread_pool [{}], type [{}], min [{}], size [{}], queue_size [{}], keep_alive [{}], wait_time [{}]", name, type, min, size, capacity.singles(), keepAlive, waitTime); return DynamicExecutors.newBlockingThreadPool(min, size, keepAlive.millis(), (int) capacity.singles(), waitTime.millis(), threadFactory); } throw new ElasticSearchIllegalArgumentException("No type found [" + type + "], for [" + name + "]"); From cd82d9ad4786ac26f80ffacb7e0a554a5d06bb69 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 11 Sep 2011 13:51:45 +0300 Subject: [PATCH 10/96] remove wrongly committed config --- config/elasticsearch.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 7bb231cf4c58c..d4c093bff6216 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -38,17 +38,3 @@ # Unicast Discovery (disable multicast) #discovery.zen.ping.multicast.enabled: false #discovery.zen.ping.unicast.hosts: ["host1", "host2"] - -threadpool: - search: - type: blocking - min: 5 - max: 20 - wait_time: 30s - keep_alive: 90s - index: - type: blocking - min: 3 - max: 10 - wait_time: 30s - keep_alive: 90s From 8fe1e5cdb89fb531e3d9f9f452f692b7bfebe143 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 10:21:10 +0300 Subject: [PATCH 11/96] upgrade to trove 3.0.0 --- modules/jarjar/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/jarjar/build.gradle b/modules/jarjar/build.gradle index 7429fbccbea28..3976f170d2b4e 100644 --- a/modules/jarjar/build.gradle +++ b/modules/jarjar/build.gradle @@ -5,7 +5,7 @@ archivesBaseName = "$project.archivesBaseName" dependencies { runtime 'com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.2' runtime 'com.google.guava:guava:r09' - runtime 'org.elasticsearch:es-trove:3.0.0rc2' + runtime 'org.elasticsearch:es-trove:3.0.0' runtime 'org.elasticsearch:es-jsr166y:20110820' runtime 'org.elasticsearch:es-jsr166e:20110820' From 93b17598046f373b844e0e65f1ed6a2e3af39040 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 11:23:20 +0300 Subject: [PATCH 12/96] Nested Mapping: Nested object with a null value causes wrong indexing structure (resulting in wrong search responses), closes #1323. --- .../index/mapper/object/ObjectMapper.java | 13 +++++----- .../mapper/nested/NestedMappingTests.java | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index a9599745e571c..a653c248840d4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -385,6 +385,13 @@ public void parse(ParseContext context) throws IOException { } XContentParser parser = context.parser(); + String currentFieldName = parser.currentName(); + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.VALUE_NULL) { + // the object is null ("obj1" : null), simply bail + return; + } + Document restoreDoc = null; if (nested.isNested()) { Document nestedDoc = new Document(); @@ -412,12 +419,6 @@ public void parse(ParseContext context) throws IOException { ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); - String currentFieldName = parser.currentName(); - XContentParser.Token token = parser.currentToken(); - if (token == XContentParser.Token.VALUE_NULL) { - // the object is null ("obj1" : null), simply bail - return; - } // if we are at the end of the previous object, advance if (token == XContentParser.Token.END_OBJECT) { token = parser.nextToken(); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java index 6fc8fb87b6fd8..624fe3c73670e 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/nested/NestedMappingTests.java @@ -33,6 +33,32 @@ @Test public class NestedMappingTests { + @Test public void emptyNested() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") + .startObject("nested1").field("type", "nested").endObject() + .endObject().endObject().endObject().string(); + + DocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .nullField("nested1") + .endObject() + .copiedBytes()); + + assertThat(doc.docs().size(), equalTo(1)); + + doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field", "value") + .startArray("nested").endArray() + .endObject() + .copiedBytes()); + + assertThat(doc.docs().size(), equalTo(1)); + } + @Test public void singleNested() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties") .startObject("nested1").field("type", "nested").endObject() From d8377074a46519f48a7d7d8b18fd4de5209b0970 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 21:33:33 +0300 Subject: [PATCH 13/96] expose meteData directory --- .../cluster/routing/allocation/RoutingAllocation.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java index 6bb6e893b337c..01a08d655de92 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/RoutingAllocation.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.routing.allocation; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingNodes; import org.elasticsearch.cluster.routing.RoutingTable; @@ -88,6 +89,10 @@ public RoutingNodes routingNodes() { return routingNodes; } + public MetaData metaData() { + return routingNodes.metaData(); + } + public DiscoveryNodes nodes() { return nodes; } From c62f476fa692d3ce27e2bdf08ad4dd345faa8d29 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 21:35:45 +0300 Subject: [PATCH 14/96] Indexation of document causes NullPointerException (on Linux) or ES process hanging, closes #1325. --- .../elasticsearch/index/mapper/object/ObjectMapper.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java index a653c248840d4..f10a4c1784d1d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/object/ObjectMapper.java @@ -427,6 +427,7 @@ public void parse(ParseContext context) throws IOException { // if we are just starting an OBJECT, advance, this is the object we are parsing, we need the name first token = parser.nextToken(); } + while (token != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.START_OBJECT) { serializeObject(context, currentFieldName); @@ -437,7 +438,7 @@ public void parse(ParseContext context) throws IOException { } else if (token == XContentParser.Token.VALUE_NULL) { serializeNullValue(context, currentFieldName); } else if (token == null) { - throw new MapperParsingException("object_mapper [" + name + "] tried to parse as object, but got EOF, has a concrete value been provided to it?"); + throw new MapperParsingException("object mapping for [" + name + "] tried to parse as object, but got EOF, has a concrete value been provided to it?"); } else if (token.isValue()) { serializeValue(context, currentFieldName, token); } @@ -480,6 +481,9 @@ private void serializeNullValue(ParseContext context, String lastFieldName) thro } private void serializeObject(final ParseContext context, String currentFieldName) throws IOException { + if (currentFieldName == null) { + throw new MapperParsingException("object mapping [" + name + "] trying to serialize an object with no field associated with it, current value [" + context.parser().textOrNull() + "]"); + } context.path().add(currentFieldName); Mapper objectMapper = mappers.get(currentFieldName); @@ -565,6 +569,9 @@ private void serializeArray(ParseContext context, String lastFieldName) throws I } private void serializeValue(final ParseContext context, String currentFieldName, XContentParser.Token token) throws IOException { + if (currentFieldName == null) { + throw new MapperParsingException("object mapping [" + name + "] trying to serialize a value with no field associated with it, current value [" + context.parser().textOrNull() + "]"); + } Mapper mapper = mappers.get(currentFieldName); if (mapper != null) { mapper.parse(context); From 36f364c99873948a1a2559abcf9d1d147040dd49 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 21:38:27 +0300 Subject: [PATCH 15/96] revert back to 5gb from 20gb (the lucene default) to create less confusion --- .../index/merge/policy/TieredMergePolicyProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/TieredMergePolicyProvider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/TieredMergePolicyProvider.java index d73917b1b226c..1c11d4371c2a9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/TieredMergePolicyProvider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/merge/policy/TieredMergePolicyProvider.java @@ -68,7 +68,7 @@ public class TieredMergePolicyProvider extends AbstractIndexShardComponent imple this.maxMergeAtOnce = componentSettings.getAsInt("max_merge_at_once", 10); this.maxMergeAtOnceExplicit = componentSettings.getAsInt("max_merge_at_once_explicit", 30); // TODO is this really a good default number for max_merge_segment, what happens for large indices, won't they end up with many segments? - this.maxMergedSegment = componentSettings.getAsBytesSize("max_merged_segment", componentSettings.getAsBytesSize("max_merge_segment", new ByteSizeValue(20, ByteSizeUnit.GB))); + this.maxMergedSegment = componentSettings.getAsBytesSize("max_merged_segment", componentSettings.getAsBytesSize("max_merge_segment", new ByteSizeValue(5, ByteSizeUnit.GB))); this.segmentsPerTier = componentSettings.getAsDouble("segments_per_tier", 10d); this.reclaimDeletesWeight = componentSettings.getAsDouble("reclaim_deletes_weight", 2.0d); From 847938a88e7893dc5bfe3e5037b49169fe797932 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 12 Sep 2011 22:44:55 +0300 Subject: [PATCH 16/96] XContentBuilder.field : StackOverflowError with Integer[] parameters, closes #1324. --- .../common/xcontent/XContentBuilder.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index 7e585255e17d5..d46f7760faa71 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -513,6 +513,7 @@ public XContentBuilder field(String name, String... value) throws IOException { return this; } + public XContentBuilder field(XContentBuilderString name, String... value) throws IOException { startArray(name); for (String o : value) { @@ -522,6 +523,24 @@ public XContentBuilder field(XContentBuilderString name, String... value) throws return this; } + public XContentBuilder field(String name, Object... value) throws IOException { + startArray(name); + for (Object o : value) { + value(o); + } + endArray(); + return this; + } + + public XContentBuilder field(XContentBuilderString name, Object... value) throws IOException { + startArray(name); + for (Object o : value) { + value(o); + } + endArray(); + return this; + } + public XContentBuilder field(String name, int... value) throws IOException { startArray(name); for (Object o : value) { From a84eb1fc8facaf72a7d46b65127fe9b664b88f0f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Sep 2011 02:15:50 +0300 Subject: [PATCH 17/96] Rivers: Close rivers early allowing them to still do index operations, closes #1330. --- .../org/elasticsearch/node/internal/InternalNode.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java index 281cb52176e02..a5dfeade20702 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -208,6 +208,9 @@ public Node start() { if (settings.getAsBoolean("http.enabled", true)) { injector.getInstance(HttpServer.class).stop(); } + + injector.getInstance(RiversManager.class).stop(); + // stop any changes happening as a result of cluster state changes injector.getInstance(IndicesClusterStateService.class).stop(); // we close indices first, so operations won't be allowed on it @@ -226,7 +229,6 @@ public Node start() { injector.getInstance(MonitorService.class).stop(); injector.getInstance(GatewayService.class).stop(); injector.getInstance(SearchService.class).stop(); - injector.getInstance(RiversManager.class).stop(); injector.getInstance(RestController.class).stop(); injector.getInstance(TransportService.class).stop(); injector.getInstance(JmxService.class).close(); @@ -256,6 +258,10 @@ public void close() { if (settings.getAsBoolean("http.enabled", true)) { injector.getInstance(HttpServer.class).close(); } + + stopWatch.stop().start("rivers"); + injector.getInstance(RiversManager.class).close(); + stopWatch.stop().start("client"); injector.getInstance(Client.class).close(); stopWatch.stop().start("indices_cluster"); @@ -277,8 +283,6 @@ public void close() { injector.getInstance(GatewayService.class).close(); stopWatch.stop().start("search"); injector.getInstance(SearchService.class).close(); - stopWatch.stop().start("indexers"); - injector.getInstance(RiversManager.class).close(); stopWatch.stop().start("rest"); injector.getInstance(RestController.class).close(); stopWatch.stop().start("transport"); From e52dbf4fdaf148cfd45b92c8301fe3d6af468ec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20Dev=C3=A8ze?= Date: Tue, 13 Sep 2011 17:57:38 +0200 Subject: [PATCH 18/96] fix bug when adding to BulkRequest with no TTL, add simple unit test for that --- .../action/bulk/BulkRequest.java | 2 +- .../action/index/IndexRequest.java | 5 ++- .../action/bulk/BulkActionTests.java | 35 +++++++++++++++++++ .../action/bulk/simple-bulk.json | 5 +++ 4 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk.json diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index c9f736a19d203..9ceeeeec30779 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -115,7 +115,7 @@ public BulkRequest add(byte[] data, int from, int length, boolean contentUnsafe) String routing = null; String parent = null; String timestamp = null; - long ttl = -1; + Long ttl = null; String opType = null; long version = 0; VersionType versionType = VersionType.INTERNAL; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java index bebebf4f6cc13..d3251492be987 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -289,7 +289,10 @@ public String timestamp() { } // Sets the relative ttl value. It musts be > 0 as it makes little sense otherwise. - public IndexRequest ttl(long ttl) throws ElasticSearchGenerationException { + public IndexRequest ttl(Long ttl) throws ElasticSearchGenerationException { + if (ttl == null) { + return this; + } if (ttl <= 0) { throw new ElasticSearchIllegalArgumentException("TTL value must be > 0. Illegal value provided [" + ttl + "]"); } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java new file mode 100644 index 0000000000000..a9cb2644e623c --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.bulk; + +import org.testng.annotations.Test; + +import static org.elasticsearch.common.io.Streams.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +public class BulkActionTests { + @Test public void testSimpleBulk() throws Exception { + String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(bulkAction.getBytes(), 0, bulkAction.length(), true); + assertThat(bulkRequest.numberOfActions(), equalTo(3)); + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk.json b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk.json new file mode 100644 index 0000000000000..b6f1b76ddf9a3 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk.json @@ -0,0 +1,5 @@ +{ "index" : { "_index" : "test", "_type" : "type1", "_id" : "1" } } +{ "field1" : "value1" } +{ "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } } +{ "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } } +{ "field1" : "value3" } From a8fc0cbcf335bc7a41498a33d77189f25a4ac42b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Sep 2011 21:43:56 +0300 Subject: [PATCH 19/96] when setting null value ttl, reset it --- .../java/org/elasticsearch/action/index/IndexRequest.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java index d3251492be987..2175d56318ec2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -288,15 +288,19 @@ public String timestamp() { return this.timestamp; } - // Sets the relative ttl value. It musts be > 0 as it makes little sense otherwise. + /** + * Sets the relative ttl value. It musts be > 0 as it makes little sense otherwise. Setting it + * to null will reset to have no ttl. + */ public IndexRequest ttl(Long ttl) throws ElasticSearchGenerationException { if (ttl == null) { + this.ttl = -1; return this; } if (ttl <= 0) { throw new ElasticSearchIllegalArgumentException("TTL value must be > 0. Illegal value provided [" + ttl + "]"); } - this.ttl = ttl; + this.ttl = ttl; return this; } From 819653cdba37be71255c39672fd28a08b9f4bc9a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 13 Sep 2011 23:33:43 +0300 Subject: [PATCH 20/96] add a note on mlockall --- config/elasticsearch.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index d4c093bff6216..febbb8a86c77c 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -9,6 +9,8 @@ #path.plugins: /path/to/plugins # Force all memory to be locked, forcing the JVM to never swap +# When setting it, make sure ES_MIN_MEM and ES_MAX_MEM are set to the same value +# and that the machine has enough memory to allocate. #bootstrap.mlockall: true # Gateway Settings From e2b1cb1640993370ba918fed319c7da12c9754da Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 14 Sep 2011 14:27:39 +0300 Subject: [PATCH 21/96] Allow to filter geo bounding box or distance based on indexed lat lon, closes #1334. --- .../geo/GeoDistanceSearchBenchmark.java | 35 +++- .../index/mapper/core/DoubleFieldMapper.java | 4 + .../index/mapper/geo/GeoPointFieldMapper.java | 37 +++- .../query/GeoBoundingBoxFilterBuilder.java | 14 ++ .../query/GeoBoundingBoxFilterParser.java | 22 ++- .../index/query/GeoDistanceFilterBuilder.java | 6 +- .../index/query/GeoDistanceFilterParser.java | 7 +- .../query/GeoDistanceRangeFilterBuilder.java | 10 ++ .../query/GeoDistanceRangeFilterParser.java | 7 +- .../index/search/geo/GeoDistance.java | 64 +++++-- .../index/search/geo/GeoDistanceFilter.java | 44 ++++- .../search/geo/GeoDistanceRangeFilter.java | 45 ++++- ...java => InMemoryGeoBoundingBoxFilter.java} | 16 +- .../geo/IndexedGeoBoundingBoxFilter.java | 159 ++++++++++++++++++ .../query/SimpleIndexQueryParserTests.java | 12 +- .../search/geo/GeoBoundingBoxTests.java | 46 ++++- .../search/geo/GeoDistanceTests.java | 33 +++- 17 files changed, 485 insertions(+), 76 deletions(-) rename modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/{GeoBoundingBoxFilter.java => InMemoryGeoBoundingBoxFilter.java} (87%) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java diff --git a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java b/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java index 402b125d73f45..71d74f9e24f33 100644 --- a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java +++ b/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/search/geo/GeoDistanceSearchBenchmark.java @@ -107,23 +107,40 @@ public static void main(String[] args) throws Exception { System.err.println("--> Warming up (ARC) - optimize_bbox"); long start = System.currentTimeMillis(); for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.ARC, true); + run(client, GeoDistance.ARC, "memory"); } long totalTime = System.currentTimeMillis() - start; - System.err.println("--> Warmup (ARC) - optimize_bbox " + (totalTime / NUM_WARM) + "ms"); + System.err.println("--> Warmup (ARC) - optimize_bbox (memory) " + (totalTime / NUM_WARM) + "ms"); - System.err.println("--> Perf (ARC) - optimize_bbox"); + System.err.println("--> Perf (ARC) - optimize_bbox (memory)"); start = System.currentTimeMillis(); for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.ARC, true); + run(client, GeoDistance.ARC, "memory"); } totalTime = System.currentTimeMillis() - start; System.err.println("--> Perf (ARC) - optimize_bbox " + (totalTime / NUM_RUNS) + "ms"); + System.err.println("--> Warming up (ARC) - optimize_bbox (indexed)"); + start = System.currentTimeMillis(); + for (int i = 0; i < NUM_WARM; i++) { + run(client, GeoDistance.ARC, "indexed"); + } + totalTime = System.currentTimeMillis() - start; + System.err.println("--> Warmup (ARC) - optimize_bbox (indexed) " + (totalTime / NUM_WARM) + "ms"); + + System.err.println("--> Perf (ARC) - optimize_bbox (indexed)"); + start = System.currentTimeMillis(); + for (int i = 0; i < NUM_RUNS; i++) { + run(client, GeoDistance.ARC, "indexed"); + } + totalTime = System.currentTimeMillis() - start; + System.err.println("--> Perf (ARC) - optimize_bbox (indexed) " + (totalTime / NUM_RUNS) + "ms"); + + System.err.println("--> Warming up (ARC) - no optimize_bbox"); start = System.currentTimeMillis(); for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.ARC, false); + run(client, GeoDistance.ARC, "none"); } totalTime = System.currentTimeMillis() - start; System.err.println("--> Warmup (ARC) - no optimize_bbox " + (totalTime / NUM_WARM) + "ms"); @@ -131,7 +148,7 @@ public static void main(String[] args) throws Exception { System.err.println("--> Perf (ARC) - no optimize_bbox"); start = System.currentTimeMillis(); for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.ARC, false); + run(client, GeoDistance.ARC, "none"); } totalTime = System.currentTimeMillis() - start; System.err.println("--> Perf (ARC) - no optimize_bbox " + (totalTime / NUM_RUNS) + "ms"); @@ -139,7 +156,7 @@ public static void main(String[] args) throws Exception { System.err.println("--> Warming up (PLANE)"); start = System.currentTimeMillis(); for (int i = 0; i < NUM_WARM; i++) { - run(client, GeoDistance.PLANE, true); + run(client, GeoDistance.PLANE, "memory"); } totalTime = System.currentTimeMillis() - start; System.err.println("--> Warmup (PLANE) " + (totalTime / NUM_WARM) + "ms"); @@ -147,7 +164,7 @@ public static void main(String[] args) throws Exception { System.err.println("--> Perf (PLANE)"); start = System.currentTimeMillis(); for (int i = 0; i < NUM_RUNS; i++) { - run(client, GeoDistance.PLANE, true); + run(client, GeoDistance.PLANE, "memory"); } totalTime = System.currentTimeMillis() - start; System.err.println("--> Perf (PLANE) " + (totalTime / NUM_RUNS) + "ms"); @@ -155,7 +172,7 @@ public static void main(String[] args) throws Exception { node.close(); } - public static void run(Client client, GeoDistance geoDistance, boolean optimizeBbox) { + public static void run(Client client, GeoDistance geoDistance, String optimizeBbox) { client.prepareSearch() // from NY .setSearchType(SearchType.COUNT) .setQuery(filteredQuery(matchAllQuery(), geoDistanceFilter("location") diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index 727af98244b8b..b927db762942b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -164,6 +164,10 @@ protected DoubleFieldMapper(Names names, int precisionStep, String fuzzyFactor, includeLower, includeUpper); } + public Filter rangeFilter(Double lowerTerm, Double upperTerm, boolean includeLower, boolean includeUpper) { + return NumericRangeFilter.newDoubleRange(names.indexName(), precisionStep, lowerTerm, upperTerm, includeLower, includeUpper); + } + @Override public Filter rangeFilter(FieldDataCache fieldDataCache, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { return NumericRangeFieldDataFilter.newDoubleRange(fieldDataCache, names.indexName(), lowerTerm == null ? null : Double.parseDouble(lowerTerm), diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java index a953c63c239e9..7a76dc94cb6d6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/geo/GeoPointFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.ObjectMapperListener; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.core.AbstractFieldMapper; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper; import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.object.ArrayValueMapperParser; @@ -147,8 +148,8 @@ public Builder store(Field.Store store) { .index(Field.Index.NOT_ANALYZED).omitNorms(true).omitTermFreqAndPositions(true).includeInAll(false).store(store).build(context); - NumberFieldMapper latMapper = null; - NumberFieldMapper lonMapper = null; + DoubleFieldMapper latMapper = null; + DoubleFieldMapper lonMapper = null; context.path().add(name); if (enableLatLon) { @@ -158,8 +159,8 @@ public Builder store(Field.Store store) { latMapperBuilder.precisionStep(precisionStep); lonMapperBuilder.precisionStep(precisionStep); } - latMapper = (NumberFieldMapper) latMapperBuilder.includeInAll(false).store(store).build(context); - lonMapper = (NumberFieldMapper) lonMapperBuilder.includeInAll(false).store(store).build(context); + latMapper = (DoubleFieldMapper) latMapperBuilder.includeInAll(false).store(store).build(context); + lonMapper = (DoubleFieldMapper) lonMapperBuilder.includeInAll(false).store(store).build(context); } StringFieldMapper geohashMapper = null; if (enableGeoHash) { @@ -226,13 +227,13 @@ public static class TypeParser implements Mapper.TypeParser { private final int precision; - private final NumberFieldMapper latMapper; + private final DoubleFieldMapper latMapper; - private final NumberFieldMapper lonMapper; + private final DoubleFieldMapper lonMapper; private final StringFieldMapper geohashMapper; - private final StringFieldMapper geoStringMapper; + private final GeoStringFieldMapper geoStringMapper; private final boolean validateLon; private final boolean validateLat; @@ -241,7 +242,7 @@ public static class TypeParser implements Mapper.TypeParser { private final boolean normalizeLat; public GeoPointFieldMapper(String name, ContentPath.Type pathType, boolean enableLatLon, boolean enableGeoHash, Integer precisionStep, int precision, - NumberFieldMapper latMapper, NumberFieldMapper lonMapper, StringFieldMapper geohashMapper, StringFieldMapper geoStringMapper, + DoubleFieldMapper latMapper, DoubleFieldMapper lonMapper, StringFieldMapper geohashMapper, GeoStringFieldMapper geoStringMapper, boolean validateLon, boolean validateLat, boolean normalizeLon, boolean normalizeLat) { this.name = name; @@ -256,6 +257,8 @@ public GeoPointFieldMapper(String name, ContentPath.Type pathType, boolean enabl this.geoStringMapper = geoStringMapper; this.geohashMapper = geohashMapper; + this.geoStringMapper.geoMapper = this; + this.validateLat = validateLat; this.validateLon = validateLon; @@ -267,6 +270,18 @@ public GeoPointFieldMapper(String name, ContentPath.Type pathType, boolean enabl return this.name; } + public DoubleFieldMapper latMapper() { + return latMapper; + } + + public DoubleFieldMapper lonMapper() { + return lonMapper; + } + + public boolean isEnableLatLon() { + return enableLatLon; + } + @Override public void parse(ParseContext context) throws IOException { ContentPath.Type origPathType = context.path().pathType(); context.path().pathType(pathType); @@ -536,6 +551,8 @@ public Builder nullValue(String nullValue) { } } + GeoPointFieldMapper geoMapper; + public GeoStringFieldMapper(Names names, Field.Index index, Field.Store store, Field.TermVector termVector, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, String nullValue, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer) { super(names, index, store, termVector, boost, omitNorms, omitTermFreqAndPositions, nullValue, indexAnalyzer, searchAnalyzer); } @@ -543,5 +560,9 @@ public GeoStringFieldMapper(Names names, Field.Index index, Field.Store store, F @Override public FieldDataType fieldDataType() { return GeoPointFieldDataType.TYPE; } + + public GeoPointFieldMapper geoMapper() { + return geoMapper; + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterBuilder.java index 506666d9f26d9..db8204c686121 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterBuilder.java @@ -44,6 +44,8 @@ public class GeoBoundingBoxFilterBuilder extends BaseFilterBuilder { private String filterName; + private String type; + public GeoBoundingBoxFilterBuilder(String name) { this.name = name; } @@ -105,6 +107,15 @@ public GeoBoundingBoxFilterBuilder cacheKey(String cacheKey) { return this; } + /** + * Sets the type of executing of the geo bounding box. Can be either `memory` or `indexed`. Defaults + * to `memory`. + */ + public GeoBoundingBoxFilterBuilder type(String type) { + this.type = type; + return this; + } + @Override protected void doXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(GeoBoundingBoxFilterParser.NAME); @@ -135,6 +146,9 @@ public GeoBoundingBoxFilterBuilder cacheKey(String cacheKey) { if (cacheKey != null) { builder.field("_cache_key", cacheKey); } + if (type != null) { + builder.field("type", type); + } builder.endObject(); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java index 842500be7688f..869fd48f9497e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoBoundingBoxFilterParser.java @@ -25,11 +25,11 @@ import org.elasticsearch.index.cache.filter.support.CacheKeyFilter; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; -import org.elasticsearch.index.search.geo.GeoBoundingBoxFilter; import org.elasticsearch.index.search.geo.GeoHashUtils; import org.elasticsearch.index.search.geo.GeoUtils; +import org.elasticsearch.index.search.geo.InMemoryGeoBoundingBoxFilter; +import org.elasticsearch.index.search.geo.IndexedGeoBoundingBoxFilter; import org.elasticsearch.index.search.geo.Point; import java.io.IOException; @@ -64,6 +64,9 @@ public class GeoBoundingBoxFilterParser implements FilterParser { XContentParser.Token token; boolean normalizeLon = true; boolean normalizeLat = true; + + String type = "memory"; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { currentFieldName = parser.currentName(); @@ -151,6 +154,8 @@ public class GeoBoundingBoxFilterParser implements FilterParser { } else if ("normalize".equals(currentFieldName)) { normalizeLat = parser.booleanValue(); normalizeLon = parser.booleanValue(); + } else if ("type".equals(currentFieldName)) { + type = parser.text(); } } } @@ -170,13 +175,22 @@ public class GeoBoundingBoxFilterParser implements FilterParser { if (mapper == null) { throw new QueryParsingException(parseContext.index(), "failed to find geo_point field [" + fieldName + "]"); } - if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) { + if (!(mapper instanceof GeoPointFieldMapper.GeoStringFieldMapper)) { throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field"); } + GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper(); + fieldName = mapper.names().indexName(); + Filter filter; + if ("indexed".equals(type)) { + filter = IndexedGeoBoundingBoxFilter.create(topLeft, bottomRight, geoMapper); + } else if ("memory".equals(type)) { + filter = new InMemoryGeoBoundingBoxFilter(topLeft, bottomRight, fieldName, parseContext.indexCache().fieldData()); + } else { + throw new QueryParsingException(parseContext.index(), "geo bounding box type [" + type + "] not supported, either 'indexed' or 'memory' are allowed"); + } - Filter filter = new GeoBoundingBoxFilter(topLeft, bottomRight, fieldName, parseContext.indexCache().fieldData()); if (cache) { filter = parseContext.cacheFilter(filter, cacheKey); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterBuilder.java index f4295cc32ae83..ef3cd4b34edc2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterBuilder.java @@ -42,7 +42,7 @@ public class GeoDistanceFilterBuilder extends BaseFilterBuilder { private GeoDistance geoDistance; - private Boolean optimizeBbox; + private String optimizeBbox; private Boolean cache; private String cacheKey; @@ -89,7 +89,7 @@ public GeoDistanceFilterBuilder geoDistance(GeoDistance geoDistance) { return this; } - public GeoDistanceFilterBuilder optimizeBbox(boolean optimizeBbox) { + public GeoDistanceFilterBuilder optimizeBbox(String optimizeBbox) { this.optimizeBbox = optimizeBbox; return this; } @@ -127,7 +127,7 @@ public GeoDistanceFilterBuilder cacheKey(String cacheKey) { builder.field("distance_type", geoDistance.name().toLowerCase()); } if (optimizeBbox != null) { - builder.field("optimize_bbox", optimizeBbox.booleanValue()); + builder.field("optimize_bbox", optimizeBbox); } if (filterName != null) { builder.field("_name", filterName); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java index 774ea8b91a166..9789d5ea880d1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceFilterParser.java @@ -74,7 +74,7 @@ public class GeoDistanceFilterParser implements FilterParser { Object vDistance = null; DistanceUnit unit = DistanceUnit.KILOMETERS; // default unit GeoDistance geoDistance = GeoDistance.ARC; - boolean optimizeBbox = true; + String optimizeBbox = "memory"; boolean normalizeLon = true; boolean normalizeLat = true; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -137,7 +137,7 @@ public class GeoDistanceFilterParser implements FilterParser { } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); } else if ("optimize_bbox".equals(currentFieldName) || "optimizeBbox".equals(currentFieldName)) { - optimizeBbox = parser.booleanValue(); + optimizeBbox = parser.textOrNull(); } else if ("normalize".equals(currentFieldName)) { normalizeLat = parser.booleanValue(); normalizeLon = parser.booleanValue(); @@ -180,9 +180,10 @@ public class GeoDistanceFilterParser implements FilterParser { if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) { throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field"); } + GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper(); fieldName = mapper.names().indexName(); - Filter filter = new GeoDistanceFilter(lat, lon, distance, geoDistance, fieldName, parseContext.indexCache().fieldData(), optimizeBbox); + Filter filter = new GeoDistanceFilter(lat, lon, distance, geoDistance, fieldName, geoMapper, parseContext.indexCache().fieldData(), optimizeBbox); if (cache) { filter = parseContext.cacheFilter(filter, cacheKey); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterBuilder.java index a4774938873ba..d3e32e2e135d0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterBuilder.java @@ -49,6 +49,8 @@ public class GeoDistanceRangeFilterBuilder extends BaseFilterBuilder { private String filterName; + private String optimizeBbox; + public GeoDistanceRangeFilterBuilder(String name) { this.name = name; } @@ -123,6 +125,11 @@ public GeoDistanceRangeFilterBuilder geoDistance(GeoDistance geoDistance) { return this; } + public GeoDistanceRangeFilterBuilder optimizeBbox(String optimizeBbox) { + this.optimizeBbox = optimizeBbox; + return this; + } + /** * Sets the filter name for the filter that can be used when searching for matched_filters per hit. */ @@ -158,6 +165,9 @@ public GeoDistanceRangeFilterBuilder cacheKey(String cacheKey) { if (geoDistance != null) { builder.field("distance_type", geoDistance.name().toLowerCase()); } + if (optimizeBbox != null) { + builder.field("optimize_bbox", optimizeBbox); + } if (filterName != null) { builder.field("_name", filterName); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java index 11268859f3d49..9615bb4b46551 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/GeoDistanceRangeFilterParser.java @@ -76,7 +76,7 @@ public class GeoDistanceRangeFilterParser implements FilterParser { boolean includeUpper = true; DistanceUnit unit = DistanceUnit.KILOMETERS; // default unit GeoDistance geoDistance = GeoDistance.ARC; - boolean optimizeBbox = true; + String optimizeBbox = "memory"; boolean normalizeLon = true; boolean normalizeLat = true; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { @@ -186,7 +186,7 @@ public class GeoDistanceRangeFilterParser implements FilterParser { } else if ("_cache_key".equals(currentFieldName) || "_cacheKey".equals(currentFieldName)) { cacheKey = new CacheKeyFilter.Key(parser.text()); } else if ("optimize_bbox".equals(currentFieldName) || "optimizeBbox".equals(currentFieldName)) { - optimizeBbox = parser.booleanValue(); + optimizeBbox = parser.textOrNull(); } else if ("normalize".equals(currentFieldName)) { normalizeLat = parser.booleanValue(); normalizeLon = parser.booleanValue(); @@ -237,9 +237,10 @@ public class GeoDistanceRangeFilterParser implements FilterParser { if (mapper.fieldDataType() != GeoPointFieldDataType.TYPE) { throw new QueryParsingException(parseContext.index(), "field [" + fieldName + "] is not a geo_point field"); } + GeoPointFieldMapper geoMapper = ((GeoPointFieldMapper.GeoStringFieldMapper) mapper).geoMapper(); fieldName = mapper.names().indexName(); - Filter filter = new GeoDistanceRangeFilter(lat, lon, from, to, includeLower, includeUpper, geoDistance, fieldName, parseContext.indexCache().fieldData(), optimizeBbox); + Filter filter = new GeoDistanceRangeFilter(lat, lon, from, to, includeLower, includeUpper, geoDistance, fieldName, geoMapper, parseContext.indexCache().fieldData(), optimizeBbox); if (cache) { filter = parseContext.cacheFilter(filter, cacheKey); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistance.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistance.java index 57fef02676598..205249704d97e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistance.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistance.java @@ -131,12 +131,12 @@ public static DistanceBoundingCheck distanceBoundingCheck(double sourceLatitude, maxLon = MAX_LON; } - Point left = new Point(Math.toDegrees(minLat), Math.toDegrees(minLon)); - Point right = new Point(Math.toDegrees(maxLat), Math.toDegrees(maxLon)); + Point topLeft = new Point(Math.toDegrees(maxLat), Math.toDegrees(minLon)); + Point bottomRight = new Point(Math.toDegrees(minLat), Math.toDegrees(maxLon)); if (minLon > maxLon) { - return new Meridian180DistanceBoundingCheck(left, right); + return new Meridian180DistanceBoundingCheck(topLeft, bottomRight); } - return new SimpleDistanceBoundingCheck(left, right); + return new SimpleDistanceBoundingCheck(topLeft, bottomRight); } public static GeoDistance fromString(String s) { @@ -158,6 +158,10 @@ public static interface FixedSourceDistance { public static interface DistanceBoundingCheck { boolean isWithin(double targetLatitude, double targetLongitude); + + Point topLeft(); + + Point bottomRight(); } public static AlwaysDistanceBoundingCheck ALWAYS_INSTANCE = new AlwaysDistanceBoundingCheck(); @@ -166,36 +170,60 @@ private static class AlwaysDistanceBoundingCheck implements DistanceBoundingChec @Override public boolean isWithin(double targetLatitude, double targetLongitude) { return true; } + + @Override public Point topLeft() { + return null; + } + + @Override public Point bottomRight() { + return null; + } } public static class Meridian180DistanceBoundingCheck implements DistanceBoundingCheck { - private final Point left; - private final Point right; + private final Point topLeft; + private final Point bottomRight; - public Meridian180DistanceBoundingCheck(Point left, Point right) { - this.left = left; - this.right = right; + public Meridian180DistanceBoundingCheck(Point topLeft, Point bottomRight) { + this.topLeft = topLeft; + this.bottomRight = bottomRight; } @Override public boolean isWithin(double targetLatitude, double targetLongitude) { - return (targetLatitude >= left.lat && targetLatitude <= right.lat) && - (targetLongitude >= left.lon || targetLongitude <= right.lon); + return (targetLatitude >= bottomRight.lat && targetLatitude <= topLeft.lat) && + (targetLongitude >= topLeft.lon || targetLongitude <= bottomRight.lon); + } + + @Override public Point topLeft() { + return topLeft; + } + + @Override public Point bottomRight() { + return bottomRight; } } public static class SimpleDistanceBoundingCheck implements DistanceBoundingCheck { - private final Point left; - private final Point right; + private final Point topLeft; + private final Point bottomRight; - public SimpleDistanceBoundingCheck(Point left, Point right) { - this.left = left; - this.right = right; + public SimpleDistanceBoundingCheck(Point topLeft, Point bottomRight) { + this.topLeft = topLeft; + this.bottomRight = bottomRight; } @Override public boolean isWithin(double targetLatitude, double targetLongitude) { - return (targetLatitude >= left.lat && targetLatitude <= right.lat) && - (targetLongitude >= left.lon && targetLongitude <= right.lon); + return (targetLatitude >= bottomRight.lat && targetLatitude <= topLeft.lat) && + (targetLongitude >= topLeft.lon && targetLongitude <= bottomRight.lon); + } + + @Override public Point topLeft() { + return topLeft; + } + + @Override public Point bottomRight() { + return bottomRight; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java index 0605d825f96fb..f25630e4dc503 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceFilter.java @@ -22,16 +22,21 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.lucene.docset.AndDocSet; +import org.elasticsearch.common.lucene.docset.DocSet; +import org.elasticsearch.common.lucene.docset.DocSets; import org.elasticsearch.common.lucene.docset.GetDocSet; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.mapper.geo.GeoPointFieldData; import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import java.io.IOException; /** - * @author kimchy (shay.banon) */ public class GeoDistanceFilter extends Filter { @@ -48,10 +53,11 @@ public class GeoDistanceFilter extends Filter { private final FieldDataCache fieldDataCache; private final GeoDistance.FixedSourceDistance fixedSourceDistance; - private final GeoDistance.DistanceBoundingCheck distanceBoundingCheck; + private GeoDistance.DistanceBoundingCheck distanceBoundingCheck; + private final Filter boundingBoxFilter; - public GeoDistanceFilter(double lat, double lon, double distance, GeoDistance geoDistance, String fieldName, FieldDataCache fieldDataCache, - boolean optimizeBbox) { + public GeoDistanceFilter(double lat, double lon, double distance, GeoDistance geoDistance, String fieldName, GeoPointFieldMapper mapper, FieldDataCache fieldDataCache, + String optimizeBbox) { this.lat = lat; this.lon = lon; this.distance = distance; @@ -60,7 +66,20 @@ public GeoDistanceFilter(double lat, double lon, double distance, GeoDistance ge this.fieldDataCache = fieldDataCache; this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, DistanceUnit.MILES); - this.distanceBoundingCheck = optimizeBbox ? GeoDistance.distanceBoundingCheck(lat, lon, distance, DistanceUnit.MILES) : GeoDistance.ALWAYS_INSTANCE; + if (optimizeBbox != null && !"none".equals(optimizeBbox)) { + distanceBoundingCheck = GeoDistance.distanceBoundingCheck(lat, lon, distance, DistanceUnit.MILES); + if ("memory".equals(optimizeBbox)) { + boundingBoxFilter = null; + } else if ("indexed".equals(optimizeBbox)) { + boundingBoxFilter = IndexedGeoBoundingBoxFilter.create(distanceBoundingCheck.topLeft(), distanceBoundingCheck.bottomRight(), mapper); + distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE; // fine, we do the bounding box check using the filter + } else { + throw new ElasticSearchIllegalArgumentException("type [" + optimizeBbox + "] for bounding box optimization not supported"); + } + } else { + distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE; + boundingBoxFilter = null; + } } public double lat() { @@ -84,8 +103,21 @@ public String fieldName() { } @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + DocSet boundingBoxDocSet = null; + if (boundingBoxFilter != null) { + DocIdSet docIdSet = boundingBoxFilter.getDocIdSet(reader); + if (docIdSet == null) { + return null; + } + boundingBoxDocSet = DocSets.convert(reader, docIdSet); + } final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName); - return new GeoDistanceDocSet(reader.maxDoc(), fieldData, fixedSourceDistance, distanceBoundingCheck, distance); + GeoDistanceDocSet distDocSet = new GeoDistanceDocSet(reader.maxDoc(), fieldData, fixedSourceDistance, distanceBoundingCheck, distance); + if (boundingBoxDocSet == null) { + return distDocSet; + } else { + return new AndDocSet(ImmutableList.of(boundingBoxDocSet, distDocSet)); + } } @Override diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java index 1e3c87eb2e2f7..cd7fd7a3b417a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoDistanceRangeFilter.java @@ -23,11 +23,17 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.lucene.docset.AndDocSet; +import org.elasticsearch.common.lucene.docset.DocSet; +import org.elasticsearch.common.lucene.docset.DocSets; import org.elasticsearch.common.lucene.docset.GetDocSet; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.index.cache.field.data.FieldDataCache; import org.elasticsearch.index.mapper.geo.GeoPointFieldData; import org.elasticsearch.index.mapper.geo.GeoPointFieldDataType; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import java.io.IOException; @@ -44,14 +50,15 @@ public class GeoDistanceRangeFilter extends Filter { private final GeoDistance geoDistance; private final GeoDistance.FixedSourceDistance fixedSourceDistance; - private final GeoDistance.DistanceBoundingCheck distanceBoundingCheck; + private GeoDistance.DistanceBoundingCheck distanceBoundingCheck; + private final Filter boundingBoxFilter; private final String fieldName; private final FieldDataCache fieldDataCache; - public GeoDistanceRangeFilter(double lat, double lon, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper, GeoDistance geoDistance, String fieldName, FieldDataCache fieldDataCache, - boolean optimizeBbox) { + public GeoDistanceRangeFilter(double lat, double lon, Double lowerVal, Double upperVal, boolean includeLower, boolean includeUpper, GeoDistance geoDistance, String fieldName, GeoPointFieldMapper mapper, FieldDataCache fieldDataCache, + String optimizeBbox) { this.lat = lat; this.lon = lon; this.geoDistance = geoDistance; @@ -73,10 +80,23 @@ public GeoDistanceRangeFilter(double lat, double lon, Double lowerVal, Double up inclusiveUpperPoint = NumericUtils.sortableLongToDouble(includeUpper ? i : (i - 1L)); } else { inclusiveUpperPoint = Double.POSITIVE_INFINITY; - optimizeBbox = false; + optimizeBbox = null; } - this.distanceBoundingCheck = optimizeBbox ? GeoDistance.distanceBoundingCheck(lat, lon, inclusiveUpperPoint, DistanceUnit.MILES) : GeoDistance.ALWAYS_INSTANCE; + if (optimizeBbox != null && !"none".equals(optimizeBbox)) { + distanceBoundingCheck = GeoDistance.distanceBoundingCheck(lat, lon, inclusiveUpperPoint, DistanceUnit.MILES); + if ("memory".equals(optimizeBbox)) { + boundingBoxFilter = null; + } else if ("indexed".equals(optimizeBbox)) { + boundingBoxFilter = IndexedGeoBoundingBoxFilter.create(distanceBoundingCheck.topLeft(), distanceBoundingCheck.bottomRight(), mapper); + distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE; // fine, we do the bounding box check using the filter + } else { + throw new ElasticSearchIllegalArgumentException("type [" + optimizeBbox + "] for bounding box optimization not supported"); + } + } else { + distanceBoundingCheck = GeoDistance.ALWAYS_INSTANCE; + boundingBoxFilter = null; + } } public double lat() { @@ -96,8 +116,21 @@ public String fieldName() { } @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { + DocSet boundingBoxDocSet = null; + if (boundingBoxFilter != null) { + DocIdSet docIdSet = boundingBoxFilter.getDocIdSet(reader); + if (docIdSet == null) { + return null; + } + boundingBoxDocSet = DocSets.convert(reader, docIdSet); + } final GeoPointFieldData fieldData = (GeoPointFieldData) fieldDataCache.cache(GeoPointFieldDataType.TYPE, reader, fieldName); - return new GeoDistanceRangeDocSet(reader.maxDoc(), fieldData, fixedSourceDistance, distanceBoundingCheck, inclusiveLowerPoint, inclusiveUpperPoint); + GeoDistanceRangeDocSet distDocSet = new GeoDistanceRangeDocSet(reader.maxDoc(), fieldData, fixedSourceDistance, distanceBoundingCheck, inclusiveLowerPoint, inclusiveUpperPoint); + if (boundingBoxDocSet == null) { + return distDocSet; + } else { + return new AndDocSet(ImmutableList.of(boundingBoxDocSet, distDocSet)); + } } @Override diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoBoundingBoxFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java similarity index 87% rename from modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoBoundingBoxFilter.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java index ea3cc61c0bcc9..11d09d25c3c05 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/GeoBoundingBoxFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/InMemoryGeoBoundingBoxFilter.java @@ -32,7 +32,7 @@ /** * @author kimchy (shay.banon) */ -public class GeoBoundingBoxFilter extends Filter { +public class InMemoryGeoBoundingBoxFilter extends Filter { private final Point topLeft; @@ -42,7 +42,7 @@ public class GeoBoundingBoxFilter extends Filter { private final FieldDataCache fieldDataCache; - public GeoBoundingBoxFilter(Point topLeft, Point bottomRight, String fieldName, FieldDataCache fieldDataCache) { + public InMemoryGeoBoundingBoxFilter(Point topLeft, Point bottomRight, String fieldName, FieldDataCache fieldDataCache) { this.topLeft = topLeft; this.bottomRight = bottomRight; this.fieldName = fieldName; @@ -66,18 +66,18 @@ public String fieldName() { //checks to see if bounding box crosses 180 degrees if (topLeft.lon > bottomRight.lon) { - return new LeftGeoBoundingBoxDocSet(reader.maxDoc(), fieldData, topLeft, bottomRight); + return new Meridian180GeoBoundingBoxDocSet(reader.maxDoc(), fieldData, topLeft, bottomRight); } else { - return new RightGeoBoundingBoxDocSet(reader.maxDoc(), fieldData, topLeft, bottomRight); + return new GeoBoundingBoxDocSet(reader.maxDoc(), fieldData, topLeft, bottomRight); } } - public static class LeftGeoBoundingBoxDocSet extends GetDocSet { + public static class Meridian180GeoBoundingBoxDocSet extends GetDocSet { private final GeoPointFieldData fieldData; private final Point topLeft; private final Point bottomRight; - public LeftGeoBoundingBoxDocSet(int maxDoc, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) { + public Meridian180GeoBoundingBoxDocSet(int maxDoc, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) { super(maxDoc); this.fieldData = fieldData; this.topLeft = topLeft; @@ -120,12 +120,12 @@ public LeftGeoBoundingBoxDocSet(int maxDoc, GeoPointFieldData fieldData, Point t } } - public static class RightGeoBoundingBoxDocSet extends GetDocSet { + public static class GeoBoundingBoxDocSet extends GetDocSet { private final GeoPointFieldData fieldData; private final Point topLeft; private final Point bottomRight; - public RightGeoBoundingBoxDocSet(int maxDoc, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) { + public GeoBoundingBoxDocSet(int maxDoc, GeoPointFieldData fieldData, Point topLeft, Point bottomRight) { super(maxDoc); this.fieldData = fieldData; this.topLeft = topLeft; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java new file mode 100644 index 0000000000000..908a6fbe230b9 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java @@ -0,0 +1,159 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.search.geo; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.Filter; +import org.apache.lucene.util.OpenBitSet; +import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; + +import java.io.IOException; + +/** + */ +public class IndexedGeoBoundingBoxFilter { + + public static Filter create(Point topLeft, Point bottomRight, GeoPointFieldMapper fieldMapper) { + if (!fieldMapper.isEnableLatLon()) { + throw new ElasticSearchIllegalArgumentException("lat/lon is not enabled (indexed) for field [" + fieldMapper.name() + "], can't use indexed filter on it"); + } + //checks to see if bounding box crosses 180 degrees + if (topLeft.lon > bottomRight.lon) { + return new LeftGeoBoundingBoxFilter(topLeft, bottomRight, fieldMapper); + } else { + return new RightGeoBoundingBoxFilter(topLeft, bottomRight, fieldMapper); + } + } + + static class LeftGeoBoundingBoxFilter extends Filter { + + final Filter lonFilter1; + final Filter lonFilter2; + final Filter latFilter; + + public LeftGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointFieldMapper fieldMapper) { + lonFilter1 = fieldMapper.lonMapper().rangeFilter(null, bottomRight.lon, true, true); + lonFilter2 = fieldMapper.lonMapper().rangeFilter(topLeft.lon, null, true, true); + latFilter = fieldMapper.latMapper().rangeFilter(bottomRight.lat, topLeft.lat, true, true); + } + + @Override public OpenBitSet getDocIdSet(IndexReader reader) throws IOException { + OpenBitSet main; + DocIdSet set = lonFilter1.getDocIdSet(reader); + if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { + main = null; + } else { + main = (OpenBitSet) set; + } + + set = lonFilter2.getDocIdSet(reader); + if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { + if (main == null) { + return null; + } else { + // nothing to do here, we remain with the main one + } + } else { + if (main == null) { + main = (OpenBitSet) set; + } else { + main.or((OpenBitSet) set); + } + } + + set = latFilter.getDocIdSet(reader); + if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { + return null; + } + main.and((OpenBitSet) set); + return main; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + LeftGeoBoundingBoxFilter that = (LeftGeoBoundingBoxFilter) o; + + if (latFilter != null ? !latFilter.equals(that.latFilter) : that.latFilter != null) return false; + if (lonFilter1 != null ? !lonFilter1.equals(that.lonFilter1) : that.lonFilter1 != null) return false; + if (lonFilter2 != null ? !lonFilter2.equals(that.lonFilter2) : that.lonFilter2 != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = lonFilter1 != null ? lonFilter1.hashCode() : 0; + result = 31 * result + (lonFilter2 != null ? lonFilter2.hashCode() : 0); + result = 31 * result + (latFilter != null ? latFilter.hashCode() : 0); + return result; + } + } + + static class RightGeoBoundingBoxFilter extends Filter { + + final Filter lonFilter; + final Filter latFilter; + + public RightGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointFieldMapper fieldMapper) { + lonFilter = fieldMapper.lonMapper().rangeFilter(topLeft.lon, bottomRight.lon, true, true); + latFilter = fieldMapper.latMapper().rangeFilter(bottomRight.lat, topLeft.lat, true, true); + } + + @Override public OpenBitSet getDocIdSet(IndexReader reader) throws IOException { + OpenBitSet main; + DocIdSet set = lonFilter.getDocIdSet(reader); + if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { + return null; + } + main = (OpenBitSet) set; + set = latFilter.getDocIdSet(reader); + if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { + return null; + } + main.and((OpenBitSet) set); + return main; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + RightGeoBoundingBoxFilter that = (RightGeoBoundingBoxFilter) o; + + if (latFilter != null ? !latFilter.equals(that.latFilter) : that.latFilter != null) return false; + if (lonFilter != null ? !lonFilter.equals(that.lonFilter) : that.lonFilter != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = lonFilter != null ? lonFilter.hashCode() : 0; + result = 31 * result + (latFilter != null ? latFilter.hashCode() : 0); + return result; + } + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java index 1563038e73a4a..a13ebb5c7591a 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/query/SimpleIndexQueryParserTests.java @@ -51,9 +51,9 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceModule; import org.elasticsearch.index.search.NumericRangeFieldDataFilter; -import org.elasticsearch.index.search.geo.GeoBoundingBoxFilter; import org.elasticsearch.index.search.geo.GeoDistanceFilter; import org.elasticsearch.index.search.geo.GeoPolygonFilter; +import org.elasticsearch.index.search.geo.InMemoryGeoBoundingBoxFilter; import org.elasticsearch.index.settings.IndexSettingsModule; import org.elasticsearch.index.similarity.SimilarityModule; import org.elasticsearch.indices.query.IndicesQueriesModule; @@ -1531,7 +1531,7 @@ private IndexQueryParserService queryParser() throws IOException { assertThat(parsedQuery.query(), instanceOf(DeletionAwareConstantScoreQuery.class)); assertThat(parsedQuery.namedFilters().containsKey("test"), equalTo(true)); DeletionAwareConstantScoreQuery constantScoreQuery = (DeletionAwareConstantScoreQuery) parsedQuery.query(); - GeoBoundingBoxFilter filter = (GeoBoundingBoxFilter) constantScoreQuery.getFilter(); + InMemoryGeoBoundingBoxFilter filter = (InMemoryGeoBoundingBoxFilter) constantScoreQuery.getFilter(); assertThat(filter.fieldName(), equalTo("location")); assertThat(filter.topLeft().lat, closeTo(40, 0.00001)); assertThat(filter.topLeft().lon, closeTo(-70, 0.00001)); @@ -1546,7 +1546,7 @@ private IndexQueryParserService queryParser() throws IOException { Query parsedQuery = queryParser.parse(query).query(); assertThat(parsedQuery, instanceOf(DeletionAwareConstantScoreQuery.class)); DeletionAwareConstantScoreQuery constantScoreQuery = (DeletionAwareConstantScoreQuery) parsedQuery; - GeoBoundingBoxFilter filter = (GeoBoundingBoxFilter) constantScoreQuery.getFilter(); + InMemoryGeoBoundingBoxFilter filter = (InMemoryGeoBoundingBoxFilter) constantScoreQuery.getFilter(); assertThat(filter.fieldName(), equalTo("location")); assertThat(filter.topLeft().lat, closeTo(40, 0.00001)); assertThat(filter.topLeft().lon, closeTo(-70, 0.00001)); @@ -1560,7 +1560,7 @@ private IndexQueryParserService queryParser() throws IOException { Query parsedQuery = queryParser.parse(query).query(); assertThat(parsedQuery, instanceOf(DeletionAwareConstantScoreQuery.class)); DeletionAwareConstantScoreQuery constantScoreQuery = (DeletionAwareConstantScoreQuery) parsedQuery; - GeoBoundingBoxFilter filter = (GeoBoundingBoxFilter) constantScoreQuery.getFilter(); + InMemoryGeoBoundingBoxFilter filter = (InMemoryGeoBoundingBoxFilter) constantScoreQuery.getFilter(); assertThat(filter.fieldName(), equalTo("location")); assertThat(filter.topLeft().lat, closeTo(40, 0.00001)); assertThat(filter.topLeft().lon, closeTo(-70, 0.00001)); @@ -1574,7 +1574,7 @@ private IndexQueryParserService queryParser() throws IOException { Query parsedQuery = queryParser.parse(query).query(); assertThat(parsedQuery, instanceOf(DeletionAwareConstantScoreQuery.class)); DeletionAwareConstantScoreQuery constantScoreQuery = (DeletionAwareConstantScoreQuery) parsedQuery; - GeoBoundingBoxFilter filter = (GeoBoundingBoxFilter) constantScoreQuery.getFilter(); + InMemoryGeoBoundingBoxFilter filter = (InMemoryGeoBoundingBoxFilter) constantScoreQuery.getFilter(); assertThat(filter.fieldName(), equalTo("location")); assertThat(filter.topLeft().lat, closeTo(40, 0.00001)); assertThat(filter.topLeft().lon, closeTo(-70, 0.00001)); @@ -1588,7 +1588,7 @@ private IndexQueryParserService queryParser() throws IOException { Query parsedQuery = queryParser.parse(query).query(); assertThat(parsedQuery, instanceOf(DeletionAwareConstantScoreQuery.class)); DeletionAwareConstantScoreQuery constantScoreQuery = (DeletionAwareConstantScoreQuery) parsedQuery; - GeoBoundingBoxFilter filter = (GeoBoundingBoxFilter) constantScoreQuery.getFilter(); + InMemoryGeoBoundingBoxFilter filter = (InMemoryGeoBoundingBoxFilter) constantScoreQuery.getFilter(); assertThat(filter.fieldName(), equalTo("location")); assertThat(filter.topLeft().lat, closeTo(40, 0.00001)); assertThat(filter.topLeft().lon, closeTo(-70, 0.00001)); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoBoundingBoxTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoBoundingBoxTests.java index 3c930d3b655ee..f26a7692e8600 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoBoundingBoxTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoBoundingBoxTests.java @@ -118,7 +118,15 @@ protected Client getClient() { assertThat(searchResponse.hits().getTotalHits(), equalTo(2l)); assertThat(searchResponse.hits().hits().length, equalTo(2)); for (SearchHit hit : searchResponse.hits()) { - System.err.println("-->" + hit.id()); + assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("5"))); + } + + searchResponse = client.prepareSearch() // from NY + .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(40.73, -74.1).bottomRight(40.717, -73.99).type("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(2l)); + assertThat(searchResponse.hits().hits().length, equalTo(2)); + for (SearchHit hit : searchResponse.hits()) { assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("5"))); } } @@ -183,6 +191,12 @@ protected Client getClient() { assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().hits().length, equalTo(1)); assertThat(searchResponse.hits().getAt(0).id(), equalTo("2")); + searchResponse = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(41, -11).bottomRight(40, 9).type("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("2")); searchResponse = client.prepareSearch() .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(41, -9).bottomRight(40, 11))) @@ -190,6 +204,12 @@ protected Client getClient() { assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().hits().length, equalTo(1)); assertThat(searchResponse.hits().getAt(0).id(), equalTo("3")); + searchResponse = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(41, -9).bottomRight(40, 11).type("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("3")); searchResponse = client.prepareSearch() .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(11, 171).bottomRight(1, -169))) @@ -197,6 +217,12 @@ protected Client getClient() { assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().hits().length, equalTo(1)); assertThat(searchResponse.hits().getAt(0).id(), equalTo("5")); + searchResponse = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(11, 171).bottomRight(1, -169).type("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("5")); searchResponse = client.prepareSearch() .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(9, 169).bottomRight(-1, -171))) @@ -204,6 +230,12 @@ protected Client getClient() { assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); assertThat(searchResponse.hits().hits().length, equalTo(1)); assertThat(searchResponse.hits().getAt(0).id(), equalTo("9")); + searchResponse = client.prepareSearch() + .setQuery(filteredQuery(matchAllQuery(), geoBoundingBoxFilter("location").topLeft(9, 169).bottomRight(-1, -171).type("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(1l)); + assertThat(searchResponse.hits().hits().length, equalTo(1)); + assertThat(searchResponse.hits().getAt(0).id(), equalTo("9")); } @Test public void limit2BoundingBoxTest() throws Exception { @@ -240,6 +272,12 @@ protected Client getClient() { geoBoundingBoxFilter("location").topLeft(74.579421999999994, 143.5).bottomRight(-66.668903999999998, 113.96875)) ).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + searchResponse = client.prepareSearch() + .setQuery( + filteredQuery(termQuery("userid", 880), + geoBoundingBoxFilter("location").topLeft(74.579421999999994, 143.5).bottomRight(-66.668903999999998, 113.96875).type("indexed")) + ).execute().actionGet(); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); searchResponse = client.prepareSearch() .setQuery( @@ -247,6 +285,12 @@ protected Client getClient() { geoBoundingBoxFilter("location").topLeft(74.579421999999994, 143.5).bottomRight(-66.668903999999998, 113.96875)) ).execute().actionGet(); assertThat(searchResponse.hits().totalHits(), equalTo(1l)); + searchResponse = client.prepareSearch() + .setQuery( + filteredQuery(termQuery("userid", 534), + geoBoundingBoxFilter("location").topLeft(74.579421999999994, 143.5).bottomRight(-66.668903999999998, 113.96875).type("indexed")) + ).execute().actionGet(); + assertThat(searchResponse.hits().totalHits(), equalTo(1l)); } } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java index eefe52dda1e09..9700255ceec55 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java @@ -38,7 +38,6 @@ import static org.hamcrest.Matchers.*; /** - * @author kimchy (shay.banon) */ public class GeoDistanceTests extends AbstractNodesTests { @@ -122,6 +121,14 @@ protected Client getClient() { for (SearchHit hit : searchResponse.hits()) { assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"), equalTo("6"))); } + searchResponse = client.prepareSearch() // from NY + .setQuery(filteredQuery(matchAllQuery(), geoDistanceFilter("location").distance("3km").point(40.7143528, -74.0059731).optimizeBbox("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(5l)); + assertThat(searchResponse.hits().hits().length, equalTo(5)); + for (SearchHit hit : searchResponse.hits()) { + assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"), equalTo("6"))); + } // now with a PLANE type searchResponse = client.prepareSearch() // from NY @@ -143,6 +150,14 @@ protected Client getClient() { for (SearchHit hit : searchResponse.hits()) { assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"))); } + searchResponse = client.prepareSearch() // from NY + .setQuery(filteredQuery(matchAllQuery(), geoDistanceFilter("location").distance("2km").point(40.7143528, -74.0059731).optimizeBbox("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.hits().hits().length, equalTo(4)); + for (SearchHit hit : searchResponse.hits()) { + assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"))); + } searchResponse = client.prepareSearch() // from NY .setQuery(filteredQuery(matchAllQuery(), geoDistanceFilter("location").distance("1.242mi").point(40.7143528, -74.0059731))) @@ -152,6 +167,14 @@ protected Client getClient() { for (SearchHit hit : searchResponse.hits()) { assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"))); } + searchResponse = client.prepareSearch() // from NY + .setQuery(filteredQuery(matchAllQuery(), geoDistanceFilter("location").distance("1.242mi").point(40.7143528, -74.0059731).optimizeBbox("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.hits().hits().length, equalTo(4)); + for (SearchHit hit : searchResponse.hits()) { + assertThat(hit.id(), anyOf(equalTo("1"), equalTo("3"), equalTo("4"), equalTo("5"))); + } searchResponse = client.prepareSearch() // from NY .setQuery(filteredQuery(matchAllQuery(), geoDistanceRangeFilter("location").from("1.0km").to("2.0km").point(40.7143528, -74.0059731))) @@ -161,6 +184,14 @@ protected Client getClient() { for (SearchHit hit : searchResponse.hits()) { assertThat(hit.id(), anyOf(equalTo("4"), equalTo("5"))); } + searchResponse = client.prepareSearch() // from NY + .setQuery(filteredQuery(matchAllQuery(), geoDistanceRangeFilter("location").from("1.0km").to("2.0km").point(40.7143528, -74.0059731).optimizeBbox("indexed"))) + .execute().actionGet(); + assertThat(searchResponse.hits().getTotalHits(), equalTo(2l)); + assertThat(searchResponse.hits().hits().length, equalTo(2)); + for (SearchHit hit : searchResponse.hits()) { + assertThat(hit.id(), anyOf(equalTo("4"), equalTo("5"))); + } // SORTING From 3eaf73a380000acef6b971fe09a7d1f38aa748d1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 14 Sep 2011 20:48:02 +0300 Subject: [PATCH 22/96] Add number of committed and number of search segments to the segments API response --- .../segments/IndicesSegmentResponse.java | 5 +++++ .../admin/indices/segments/ShardSegments.java | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index ef1b0541c8953..c36f0cacdd0cd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -118,6 +118,9 @@ public Map indices() { } builder.endObject(); + builder.field(Fields.NUM_COMMITTED_SEGMENTS, shardSegments.numberOfCommitted()); + builder.field(Fields.NUM_SEARCH_SEGMENTS, shardSegments.numberOfSearch()); + builder.startObject(Fields.SEGMENTS); for (Segment segment : shardSegments) { builder.startObject(segment.name()); @@ -156,6 +159,8 @@ static final class Fields { static final XContentBuilderString SEGMENTS = new XContentBuilderString("segments"); static final XContentBuilderString GENERATION = new XContentBuilderString("generation"); + static final XContentBuilderString NUM_COMMITTED_SEGMENTS = new XContentBuilderString("num_committed_segments"); + static final XContentBuilderString NUM_SEARCH_SEGMENTS = new XContentBuilderString("num_search_segments"); static final XContentBuilderString NUM_DOCS = new XContentBuilderString("num_docs"); static final XContentBuilderString DELETED_DOCS = new XContentBuilderString("deleted_docs"); static final XContentBuilderString SIZE = new XContentBuilderString("size"); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java index c2181a4b0d966..a96660077f182 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java @@ -68,6 +68,26 @@ public List getSegments() { return segments; } + public int numberOfCommitted() { + int count = 0; + for (Segment segment : segments) { + if (segment.committed()) { + count++; + } + } + return count; + } + + public int numberOfSearch() { + int count = 0; + for (Segment segment : segments) { + if (segment.search()) { + count++; + } + } + return count; + } + public static ShardSegments readShardSegments(StreamInput in) throws IOException { ShardSegments shard = new ShardSegments(); shard.readFrom(in); From 74079c8a130986d2c8a5b8765b756c9a059d3b42 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Sep 2011 11:50:59 +0300 Subject: [PATCH 23/96] Upgrade to Lucene 3.4.0, closes #1335. --- .idea/libraries/lucene.xml | 20 +- .idea/modules/plugin-analysis-icu.iml | 4 +- modules/elasticsearch/build.gradle | 10 +- .../analysis/synonym/SynonymFilter.java | 258 ------------- .../lucene/analysis/synonym/SynonymMap.java | 177 --------- .../lucene/search/PublicTermsFilter.java | 10 +- .../elasticsearch/common/lucene/Lucene.java | 11 +- .../common/lucene/docset/DocSets.java | 100 ++++- .../common/lucene/docset/FixedBitDocSet.java | 62 ++++ .../lucene/docset/SlicedOpenBitSet.java | 344 ------------------ .../common/lucene/search/TermFilter.java | 10 +- .../common/lucene/search/XBooleanFilter.java | 51 +-- .../common/lucene/uid/UidField.java | 7 +- .../analysis/SynonymTokenFilterFactory.java | 239 ++++++++---- .../AbstractConcurrentMapFilterCache.java | 33 +- .../support/AbstractWeightedFilterCache.java | 33 +- .../filter/support/FilterCacheValue.java | 40 +- .../mapper/core/AbstractFieldMapper.java | 11 +- .../index/mapper/core/NumberFieldMapper.java | 3 +- .../elasticsearch/index/search/UidFilter.java | 8 +- .../index/search/child/ChildCollector.java | 14 +- .../index/search/child/HasChildFilter.java | 4 +- .../geo/IndexedGeoBoundingBoxFilter.java | 23 +- .../index/search/nested/BlockJoinQuery.java | 18 +- .../nested/NestedChildrenCollector.java | 8 +- .../search/nested/NonNestedDocsFilter.java | 6 +- .../lucene/docset/SlicedOpenBitSetTests.java | 54 --- .../deps/lucene/SimpleLuceneTests.java | 52 +-- plugins/analysis/icu/build.gradle | 6 +- 29 files changed, 457 insertions(+), 1159 deletions(-) delete mode 100644 modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymFilter.java delete mode 100644 modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymMap.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/FixedBitDocSet.java delete mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSet.java delete mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSetTests.java diff --git a/.idea/libraries/lucene.xml b/.idea/libraries/lucene.xml index 83be9135aab0e..60fb9b0fe593c 100644 --- a/.idea/libraries/lucene.xml +++ b/.idea/libraries/lucene.xml @@ -1,19 +1,19 @@ - - - - - + + + + + - - - - - + + + + + \ No newline at end of file diff --git a/.idea/modules/plugin-analysis-icu.iml b/.idea/modules/plugin-analysis-icu.iml index 7eace2e1008c6..8815b515a42d1 100644 --- a/.idea/modules/plugin-analysis-icu.iml +++ b/.idea/modules/plugin-analysis-icu.iml @@ -19,11 +19,11 @@ - + - + diff --git a/modules/elasticsearch/build.gradle b/modules/elasticsearch/build.gradle index 90c6e81240bcb..abfc0d0654dc1 100644 --- a/modules/elasticsearch/build.gradle +++ b/modules/elasticsearch/build.gradle @@ -38,11 +38,11 @@ dependencies { compile('net.java.dev.jna:jna:3.2.7') { transitive = false } - compile('org.apache.lucene:lucene-core:3.3.0') { transitive = false } - compile('org.apache.lucene:lucene-analyzers:3.3.0') { transitive = false } - compile('org.apache.lucene:lucene-queries:3.3.0') { transitive = false } - compile('org.apache.lucene:lucene-memory:3.3.0') { transitive = false } - compile('org.apache.lucene:lucene-highlighter:3.3.0') { transitive = false } + compile('org.apache.lucene:lucene-core:3.4.0') { transitive = false } + compile('org.apache.lucene:lucene-analyzers:3.4.0') { transitive = false } + compile('org.apache.lucene:lucene-queries:3.4.0') { transitive = false } + compile('org.apache.lucene:lucene-memory:3.4.0') { transitive = false } + compile('org.apache.lucene:lucene-highlighter:3.4.0') { transitive = false } } configurations { diff --git a/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymFilter.java b/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymFilter.java deleted file mode 100644 index c55fe17d8eff5..0000000000000 --- a/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymFilter.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lucene.analysis.synonym; - -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TypeAttribute; -import org.apache.lucene.util.AttributeSource; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.LinkedList; - -/** - * SynonymFilter handles multi-token synonyms with variable position increment offsets. - *

- * The matched tokens from the input stream may be optionally passed through (includeOrig=true) - * or discarded. If the original tokens are included, the position increments may be modified - * to retain absolute positions after merging with the synonym tokenstream. - *

- * Generated synonyms will start at the same position as the first matched source token. - */ -// LUCENE MONITOR: Taken from 4.0, remove once upgraded -public final class SynonymFilter extends TokenFilter { - - private final SynonymMap map; // Map - private Iterator replacement; // iterator over generated tokens - - public SynonymFilter(TokenStream in, SynonymMap map) { - super(in); - if (map == null) - throw new IllegalArgumentException("map is required"); - - this.map = map; - // just ensuring these attributes exist... - addAttribute(CharTermAttribute.class); - addAttribute(PositionIncrementAttribute.class); - addAttribute(OffsetAttribute.class); - addAttribute(TypeAttribute.class); - } - - - /* - * Need to worry about multiple scenarios: - * - need to go for the longest match - * a b => foo #shouldn't match if "a b" is followed by "c d" - * a b c d => bar - * - need to backtrack - retry matches for tokens already read - * a b c d => foo - * b c => bar - * If the input stream is "a b c x", one will consume "a b c d" - * trying to match the first rule... all but "a" should be - * pushed back so a match may be made on "b c". - * - don't try and match generated tokens (thus need separate queue) - * matching is not recursive. - * - handle optional generation of original tokens in all these cases, - * merging token streams to preserve token positions. - * - preserve original positionIncrement of first matched token - */ - @Override - public boolean incrementToken() throws IOException { - while (true) { - // if there are any generated tokens, return them... don't try any - // matches against them, as we specifically don't want recursion. - if (replacement != null && replacement.hasNext()) { - copy(this, replacement.next()); - return true; - } - - // common case fast-path of first token not matching anything - AttributeSource firstTok = nextTok(); - if (firstTok == null) return false; - CharTermAttribute termAtt = firstTok.addAttribute(CharTermAttribute.class); - SynonymMap result = map.submap != null ? map.submap.get(termAtt.buffer(), 0, termAtt.length()) : null; - if (result == null) { - copy(this, firstTok); - return true; - } - - // fast-path failed, clone ourselves if needed - if (firstTok == this) - firstTok = cloneAttributes(); - // OK, we matched a token, so find the longest match. - - matched = new LinkedList(); - - result = match(result); - - if (result == null) { - // no match, simply return the first token read. - copy(this, firstTok); - return true; - } - - // reuse, or create new one each time? - ArrayList generated = new ArrayList(result.synonyms.length + matched.size() + 1); - - // - // there was a match... let's generate the new tokens, merging - // in the matched tokens (position increments need adjusting) - // - AttributeSource lastTok = matched.isEmpty() ? firstTok : matched.getLast(); - boolean includeOrig = result.includeOrig(); - - AttributeSource origTok = includeOrig ? firstTok : null; - PositionIncrementAttribute firstPosIncAtt = firstTok.addAttribute(PositionIncrementAttribute.class); - int origPos = firstPosIncAtt.getPositionIncrement(); // position of origTok in the original stream - int repPos = 0; // curr position in replacement token stream - int pos = 0; // current position in merged token stream - - for (int i = 0; i < result.synonyms.length; i++) { - Token repTok = result.synonyms[i]; - AttributeSource newTok = firstTok.cloneAttributes(); - CharTermAttribute newTermAtt = newTok.addAttribute(CharTermAttribute.class); - OffsetAttribute newOffsetAtt = newTok.addAttribute(OffsetAttribute.class); - PositionIncrementAttribute newPosIncAtt = newTok.addAttribute(PositionIncrementAttribute.class); - - OffsetAttribute lastOffsetAtt = lastTok.addAttribute(OffsetAttribute.class); - - newOffsetAtt.setOffset(newOffsetAtt.startOffset(), lastOffsetAtt.endOffset()); - newTermAtt.copyBuffer(repTok.buffer(), 0, repTok.length()); - repPos += repTok.getPositionIncrement(); - if (i == 0) repPos = origPos; // make position of first token equal to original - - // if necessary, insert original tokens and adjust position increment - while (origTok != null && origPos <= repPos) { - PositionIncrementAttribute origPosInc = origTok.addAttribute(PositionIncrementAttribute.class); - origPosInc.setPositionIncrement(origPos - pos); - generated.add(origTok); - pos += origPosInc.getPositionIncrement(); - origTok = matched.isEmpty() ? null : matched.removeFirst(); - if (origTok != null) { - origPosInc = origTok.addAttribute(PositionIncrementAttribute.class); - origPos += origPosInc.getPositionIncrement(); - } - } - - newPosIncAtt.setPositionIncrement(repPos - pos); - generated.add(newTok); - pos += newPosIncAtt.getPositionIncrement(); - } - - // finish up any leftover original tokens - while (origTok != null) { - PositionIncrementAttribute origPosInc = origTok.addAttribute(PositionIncrementAttribute.class); - origPosInc.setPositionIncrement(origPos - pos); - generated.add(origTok); - pos += origPosInc.getPositionIncrement(); - origTok = matched.isEmpty() ? null : matched.removeFirst(); - if (origTok != null) { - origPosInc = origTok.addAttribute(PositionIncrementAttribute.class); - origPos += origPosInc.getPositionIncrement(); - } - } - - // what if we replaced a longer sequence with a shorter one? - // a/0 b/5 => foo/0 - // should I re-create the gap on the next buffered token? - - replacement = generated.iterator(); - // Now return to the top of the loop to read and return the first - // generated token.. The reason this is done is that we may have generated - // nothing at all, and may need to continue with more matching logic. - } - } - - - // - // Defer creation of the buffer until the first time it is used to - // optimize short fields with no matches. - // - private LinkedList buffer; - private LinkedList matched; - - private AttributeSource nextTok() throws IOException { - if (buffer != null && !buffer.isEmpty()) { - return buffer.removeFirst(); - } else { - if (input.incrementToken()) { - return this; - } else - return null; - } - } - - private void pushTok(AttributeSource t) { - if (buffer == null) buffer = new LinkedList(); - buffer.addFirst(t); - } - - private SynonymMap match(SynonymMap map) throws IOException { - SynonymMap result = null; - - if (map.submap != null) { - AttributeSource tok = nextTok(); - if (tok != null) { - // clone ourselves. - if (tok == this) - tok = cloneAttributes(); - // check for positionIncrement!=1? if>1, should not match, if==0, check multiple at this level? - CharTermAttribute termAtt = tok.getAttribute(CharTermAttribute.class); - SynonymMap subMap = map.submap.get(termAtt.buffer(), 0, termAtt.length()); - - if (subMap != null) { - // recurse - result = match(subMap); - } - - if (result != null) { - matched.addFirst(tok); - } else { - // push back unmatched token - pushTok(tok); - } - } - } - - // if no longer sequence matched, so if this node has synonyms, it's the match. - if (result == null && map.synonyms != null) { - result = map; - } - - return result; - } - - private void copy(AttributeSource target, AttributeSource source) { - if (target != source) - source.copyTo(target); - } - - @Override - public void reset() throws IOException { - input.reset(); - replacement = null; - } -} diff --git a/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymMap.java b/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymMap.java deleted file mode 100644 index ab654a9d8362c..0000000000000 --- a/modules/elasticsearch/src/main/java/org/apache/lucene/analysis/synonym/SynonymMap.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.lucene.analysis.synonym; - -import org.apache.lucene.analysis.CharArrayMap; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.util.Version; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -/** - * Mapping rules for use with {@link SynonymFilter} - */ -public class SynonymMap { - /** - * @lucene.internal - */ - public CharArrayMap submap; // recursive: Map - /** - * @lucene.internal - */ - public Token[] synonyms; - int flags; - - static final int INCLUDE_ORIG = 0x01; - static final int IGNORE_CASE = 0x02; - - public SynonymMap() { - } - - public SynonymMap(boolean ignoreCase) { - if (ignoreCase) flags |= IGNORE_CASE; - } - - public boolean includeOrig() { - return (flags & INCLUDE_ORIG) != 0; - } - - public boolean ignoreCase() { - return (flags & IGNORE_CASE) != 0; - } - - /** - * @param singleMatch List, the sequence of strings to match - * @param replacement List the list of tokens to use on a match - * @param includeOrig sets a flag on this mapping signaling the generation of matched tokens in addition to the replacement tokens - * @param mergeExisting merge the replacement tokens with any other mappings that exist - */ - public void add(List singleMatch, List replacement, boolean includeOrig, boolean mergeExisting) { - SynonymMap currMap = this; - for (String str : singleMatch) { - if (currMap.submap == null) { - // for now hardcode at 4.0, as its what the old code did. - // would be nice to fix, but shouldn't store a version in each submap!!! - currMap.submap = new CharArrayMap(Version.LUCENE_31, 1, ignoreCase()); - } - - SynonymMap map = currMap.submap.get(str); - if (map == null) { - map = new SynonymMap(); - map.flags |= flags & IGNORE_CASE; - currMap.submap.put(str, map); - } - - currMap = map; - } - - if (currMap.synonyms != null && !mergeExisting) { - throw new RuntimeException("SynonymFilter: there is already a mapping for " + singleMatch); - } - List superset = currMap.synonyms == null ? replacement : - mergeTokens(Arrays.asList(currMap.synonyms), replacement); - currMap.synonyms = superset.toArray(new Token[superset.size()]); - if (includeOrig) currMap.flags |= INCLUDE_ORIG; - } - - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("<"); - if (synonyms != null) { - sb.append("["); - for (int i = 0; i < synonyms.length; i++) { - if (i != 0) sb.append(','); - sb.append(synonyms[i]); - } - if ((flags & INCLUDE_ORIG) != 0) { - sb.append(",ORIG"); - } - sb.append("],"); - } - sb.append(submap); - sb.append(">"); - return sb.toString(); - } - - - /** - * Produces a List from a List - */ - public static List makeTokens(List strings) { - List ret = new ArrayList(strings.size()); - for (String str : strings) { - //Token newTok = new Token(str,0,0,"SYNONYM"); - Token newTok = new Token(str, 0, 0, "SYNONYM"); - ret.add(newTok); - } - return ret; - } - - - /** - * Merge two lists of tokens, producing a single list with manipulated positionIncrements so that - * the tokens end up at the same position. - * - * Example: [a b] merged with [c d] produces [a/b c/d] ('/' denotes tokens in the same position) - * Example: [a,5 b,2] merged with [c d,4 e,4] produces [c a,5/d b,2 e,2] (a,n means a has posInc=n) - */ - public static List mergeTokens(List lst1, List lst2) { - ArrayList result = new ArrayList(); - if (lst1 == null || lst2 == null) { - if (lst2 != null) result.addAll(lst2); - if (lst1 != null) result.addAll(lst1); - return result; - } - - int pos = 0; - Iterator iter1 = lst1.iterator(); - Iterator iter2 = lst2.iterator(); - Token tok1 = iter1.hasNext() ? iter1.next() : null; - Token tok2 = iter2.hasNext() ? iter2.next() : null; - int pos1 = tok1 != null ? tok1.getPositionIncrement() : 0; - int pos2 = tok2 != null ? tok2.getPositionIncrement() : 0; - while (tok1 != null || tok2 != null) { - while (tok1 != null && (pos1 <= pos2 || tok2 == null)) { - Token tok = new Token(tok1.startOffset(), tok1.endOffset(), tok1.type()); - tok.copyBuffer(tok1.buffer(), 0, tok1.length()); - tok.setPositionIncrement(pos1 - pos); - result.add(tok); - pos = pos1; - tok1 = iter1.hasNext() ? iter1.next() : null; - pos1 += tok1 != null ? tok1.getPositionIncrement() : 0; - } - while (tok2 != null && (pos2 <= pos1 || tok1 == null)) { - Token tok = new Token(tok2.startOffset(), tok2.endOffset(), tok2.type()); - tok.copyBuffer(tok2.buffer(), 0, tok2.length()); - tok.setPositionIncrement(pos2 - pos); - result.add(tok); - pos = pos2; - tok2 = iter2.hasNext() ? iter2.next() : null; - pos2 += tok2 != null ? tok2.getPositionIncrement() : 0; - } - } - return result; - } - -} diff --git a/modules/elasticsearch/src/main/java/org/apache/lucene/search/PublicTermsFilter.java b/modules/elasticsearch/src/main/java/org/apache/lucene/search/PublicTermsFilter.java index b8a9d54f2aa6a..fcc7b6138cced 100644 --- a/modules/elasticsearch/src/main/java/org/apache/lucene/search/PublicTermsFilter.java +++ b/modules/elasticsearch/src/main/java/org/apache/lucene/search/PublicTermsFilter.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import java.io.IOException; import java.util.Iterator; @@ -73,18 +73,18 @@ public int hashCode() { @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { - OpenBitSet result = null; + FixedBitSet result = null; TermDocs td = reader.termDocs(); try { for (Term term : terms) { td.seek(term); if (td.next()) { if (result == null) { - result = new OpenBitSet(reader.maxDoc()); + result = new FixedBitSet(reader.maxDoc()); } - result.fastSet(td.doc()); + result.set(td.doc()); while (td.next()) { - result.fastSet(td.doc()); + result.set(td.doc()); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/Lucene.java index e02ab89776684..bee40a036b75d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -49,7 +49,7 @@ */ public class Lucene { - public static final Version VERSION = Version.LUCENE_31; + public static final Version VERSION = Version.LUCENE_34; public static final Version ANALYZER_VERSION = VERSION; public static final Version QUERYPARSER_VERSION = VERSION; @@ -62,6 +62,15 @@ public static Version parseVersion(@Nullable String version, Version defaultVers if (version == null) { return defaultVersion; } + if ("3.4".equals(version)) { + return Version.LUCENE_34; + } + if ("3.3".equals(version)) { + return Version.LUCENE_33; + } + if ("3.2".equals(version)) { + return Version.LUCENE_32; + } if ("3.1".equals(version)) { return Version.LUCENE_31; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSets.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSets.java index def3c9fd2f44c..cc48a05e94b35 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSets.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/DocSets.java @@ -22,20 +22,112 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.OpenBitSet; import java.io.IOException; /** - * @author kimchy (Shay Banon) */ public class DocSets { + public static FixedBitSet createFixedBitSet(DocIdSetIterator disi, int numBits) throws IOException { + FixedBitSet set = new FixedBitSet(numBits); + int doc; + while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + set.set(doc); + } + return set; + } + + public static void or(FixedBitSet into, DocIdSet other) throws IOException { + if (other == null) { + return; + } + if (other instanceof FixedBitSet) { + into.or((FixedBitSet) other); + } else if (other instanceof FixedBitDocSet) { + into.or(((FixedBitDocSet) other).set()); + } else { + DocIdSetIterator disi = other.iterator(); + if (disi != null) { + into.or(disi); + } + } + } + + public static void and(FixedBitSet into, DocIdSet other) throws IOException { + if (other instanceof FixedBitDocSet) { + other = ((FixedBitDocSet) other).set(); + } + if (other instanceof FixedBitSet) { + // copied from OpenBitSet#and + long[] intoBits = into.getBits(); + long[] otherBits = ((FixedBitSet) other).getBits(); + assert intoBits.length == otherBits.length; + // testing against zero can be more efficient + int pos = intoBits.length; + while (--pos >= 0) { + intoBits[pos] &= otherBits[pos]; + } + } else { + if (other == null) { + into.clear(0, into.length() + 1); + } else { + // copied from OpenBitSetDISI#inPlaceAnd + DocIdSetIterator disi = other.iterator(); + if (disi == null) { + into.clear(0, into.length() + 1); + } else { + int bitSetDoc = into.nextSetBit(0); + int disiDoc; + while (bitSetDoc != -1 && (disiDoc = disi.advance(bitSetDoc)) != DocIdSetIterator.NO_MORE_DOCS) { + into.clear(bitSetDoc, disiDoc); + bitSetDoc = into.nextSetBit(disiDoc + 1); + } + if (bitSetDoc != -1) { + into.clear(bitSetDoc, into.length() + 1); + } + } + } + } + } + + public static void andNot(FixedBitSet into, DocIdSet other) throws IOException { + if (other == null) { + return; + } + if (other instanceof FixedBitDocSet) { + other = ((FixedBitDocSet) other).set(); + } + if (other instanceof FixedBitSet) { + // copied from OpenBitSet#andNot + long[] intoBits = into.getBits(); + long[] otherBits = ((FixedBitSet) other).getBits(); + assert intoBits.length == otherBits.length; + int idx = intoBits.length; + while (--idx >= 0) { + intoBits[idx] &= ~otherBits[idx]; + } + } else { + // copied from OpenBitSetDISI#inPlaceNot + DocIdSetIterator disi = other.iterator(); + if (disi != null) { + int doc; + while ((doc = disi.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + into.clear(doc); + } + } + } + } + public static DocSet convert(IndexReader reader, DocIdSet docIdSet) throws IOException { if (docIdSet == null) { return DocSet.EMPTY_DOC_SET; } else if (docIdSet instanceof DocSet) { return (DocSet) docIdSet; + } else if (docIdSet instanceof FixedBitSet) { + return new FixedBitDocSet((FixedBitSet) docIdSet); } else if (docIdSet instanceof OpenBitSet) { return new OpenBitDocSet((OpenBitSet) docIdSet); } else { @@ -43,7 +135,7 @@ public static DocSet convert(IndexReader reader, DocIdSet docIdSet) throws IOExc // null is allowed to be returned by iterator(), // in this case we wrap with the empty set, // which is cacheable. - return (it == null) ? DocSet.EMPTY_DOC_SET : new OpenBitDocSet(it, reader.maxDoc()); + return (it == null) ? DocSet.EMPTY_DOC_SET : new FixedBitDocSet(createFixedBitSet(it, reader.maxDoc())); } } @@ -55,6 +147,8 @@ public static DocSet cacheable(IndexReader reader, DocIdSet docIdSet) throws IOE return DocSet.EMPTY_DOC_SET; } else if (docIdSet.isCacheable() && (docIdSet instanceof DocSet)) { return (DocSet) docIdSet; + } else if (docIdSet instanceof FixedBitSet) { + return new FixedBitDocSet((FixedBitSet) docIdSet); } else if (docIdSet instanceof OpenBitSet) { return new OpenBitDocSet((OpenBitSet) docIdSet); } else { @@ -62,7 +156,7 @@ public static DocSet cacheable(IndexReader reader, DocIdSet docIdSet) throws IOE // null is allowed to be returned by iterator(), // in this case we wrap with the empty set, // which is cacheable. - return (it == null) ? DocSet.EMPTY_DOC_SET : new OpenBitDocSet(it, reader.maxDoc()); + return (it == null) ? DocSet.EMPTY_DOC_SET : new FixedBitDocSet(createFixedBitSet(it, reader.maxDoc())); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/FixedBitDocSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/FixedBitDocSet.java new file mode 100644 index 0000000000000..516d5664b5e76 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/FixedBitDocSet.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.lucene.docset; + +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.common.RamUsage; + +import java.io.IOException; + +/** + * @author kimchy (shay.banon) + */ +public class FixedBitDocSet extends DocSet { + + private final FixedBitSet set; + + public FixedBitDocSet(FixedBitSet set) { + this.set = set; + } + + public FixedBitDocSet(int numBits) { + this.set = new FixedBitSet(numBits); + } + + @Override public boolean isCacheable() { + return true; + } + + public FixedBitSet set() { + return set; + } + + @Override public boolean get(int doc) throws IOException { + return set.get(doc); + } + + @Override public DocIdSetIterator iterator() throws IOException { + return set.iterator(); + } + + @Override public long sizeInBytes() { + return set.getBits().length * RamUsage.NUM_BYTES_LONG + RamUsage.NUM_BYTES_ARRAY_HEADER + RamUsage.NUM_BYTES_INT /* wlen */; + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSet.java deleted file mode 100644 index d41205e3b81c8..0000000000000 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSet.java +++ /dev/null @@ -1,344 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.lucene.docset; - -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BitUtil; -import org.apache.lucene.util.OpenBitSet; -import org.elasticsearch.common.RamUsage; - -import java.io.IOException; - -/** - * Derived from {@link org.apache.lucene.util.OpenBitSet} but works from a slice out of a provided long[] array. - * It does not expand, as it assumes that the slice is from a cached long[] array, so we can't really expand... - */ -public class SlicedOpenBitSet extends DocSet { - - private final long[] bits; - private final int wlen; // number of words (elements) used in the array - private final int from; // the from index in the array - - public SlicedOpenBitSet(long[] bits, int from, OpenBitSet setToCopy) { - this.bits = bits; - this.from = from; - System.arraycopy(setToCopy.getBits(), 0, bits, from, setToCopy.getBits().length); - this.wlen = setToCopy.getNumWords(); - } - - public SlicedOpenBitSet(long[] bits, int wlen, int from) { - this.bits = bits; - this.wlen = wlen; - this.from = from; - } - - @Override public boolean isCacheable() { - return true; - } - - @Override public long sizeInBytes() { - return wlen * RamUsage.NUM_BYTES_LONG + RamUsage.NUM_BYTES_ARRAY_HEADER + RamUsage.NUM_BYTES_INT /* wlen */; - } - - /** - * Returns the current capacity in bits (1 greater than the index of the last bit) - */ - public long capacity() { - return (bits.length - from) << 6; - } - - /** - * Returns the current capacity of this set. Included for - * compatibility. This is *not* equal to {@link #cardinality} - */ - public long size() { - return capacity(); - } - - /** - * @return the number of set bits - */ - public long cardinality() { - return BitUtil.pop_array(bits, from, wlen); - } - - /** - * Returns true or false for the specified bit index. - */ - public boolean get(int index) { - return fastGet(index); -// int i = index >> 6; // div 64 -// // signed shift will keep a negative index and force an -// // array-index-out-of-bounds-exception, removing the need for an explicit check. -// if (from + i >= wlen) return false; -// -// int bit = index & 0x3f; // mod 64 -// long bitmask = 1L << bit; -// return (bits[from + i] & bitmask) != 0; - } - - /** - * Returns true or false for the specified bit index. - * The index should be less than the OpenBitSet size - */ - public boolean fastGet(int index) { - int i = index >> 6; // div 64 - // signed shift will keep a negative index and force an - // array-index-out-of-bounds-exception, removing the need for an explicit check. - int bit = index & 0x3f; // mod 64 - long bitmask = 1L << bit; - return (bits[from + i] & bitmask) != 0; - } - - - /** - * Returns true or false for the specified bit index - */ - public boolean get(long index) { - int i = (int) (index >> 6); // div 64 - if (from + i >= wlen) return false; - int bit = (int) index & 0x3f; // mod 64 - long bitmask = 1L << bit; - return (bits[from + i] & bitmask) != 0; - } - - /** - * Returns true or false for the specified bit index. - * The index should be less than the OpenBitSet size. - */ - public boolean fastGet(long index) { - int i = (int) (index >> 6); // div 64 - int bit = (int) index & 0x3f; // mod 64 - long bitmask = 1L << bit; - return (bits[from + i] & bitmask) != 0; - } - - /** - * Sets the bit at the specified index. - * The index should be less than the OpenBitSet size. - */ - public void fastSet(int index) { - int wordNum = index >> 6; // div 64 - int bit = index & 0x3f; // mod 64 - long bitmask = 1L << bit; - bits[from + wordNum] |= bitmask; - } - - /** - * Sets the bit at the specified index. - * The index should be less than the OpenBitSet size. - */ - public void fastSet(long index) { - int wordNum = (int) (index >> 6); - int bit = (int) index & 0x3f; - long bitmask = 1L << bit; - bits[from + wordNum] |= bitmask; - } - - @Override public DocIdSetIterator iterator() throws IOException { - return new SlicedIterator(this); - } - - - /** - * An iterator to iterate over set bits in an OpenBitSet. - * This is faster than nextSetBit() for iterating over the complete set of bits, - * especially when the density of the bits set is high. - */ - public static class SlicedIterator extends DocIdSetIterator { - - // The General Idea: instead of having an array per byte that has - // the offsets of the next set bit, that array could be - // packed inside a 32 bit integer (8 4 bit numbers). That - // should be faster than accessing an array for each index, and - // the total array size is kept smaller (256*sizeof(int))=1K - protected final static int[] bitlist = { - 0x0, 0x1, 0x2, 0x21, 0x3, 0x31, 0x32, 0x321, 0x4, 0x41, 0x42, 0x421, 0x43, - 0x431, 0x432, 0x4321, 0x5, 0x51, 0x52, 0x521, 0x53, 0x531, 0x532, 0x5321, - 0x54, 0x541, 0x542, 0x5421, 0x543, 0x5431, 0x5432, 0x54321, 0x6, 0x61, 0x62, - 0x621, 0x63, 0x631, 0x632, 0x6321, 0x64, 0x641, 0x642, 0x6421, 0x643, - 0x6431, 0x6432, 0x64321, 0x65, 0x651, 0x652, 0x6521, 0x653, 0x6531, 0x6532, - 0x65321, 0x654, 0x6541, 0x6542, 0x65421, 0x6543, 0x65431, 0x65432, 0x654321, - 0x7, 0x71, 0x72, 0x721, 0x73, 0x731, 0x732, 0x7321, 0x74, 0x741, 0x742, - 0x7421, 0x743, 0x7431, 0x7432, 0x74321, 0x75, 0x751, 0x752, 0x7521, 0x753, - 0x7531, 0x7532, 0x75321, 0x754, 0x7541, 0x7542, 0x75421, 0x7543, 0x75431, - 0x75432, 0x754321, 0x76, 0x761, 0x762, 0x7621, 0x763, 0x7631, 0x7632, - 0x76321, 0x764, 0x7641, 0x7642, 0x76421, 0x7643, 0x76431, 0x76432, 0x764321, - 0x765, 0x7651, 0x7652, 0x76521, 0x7653, 0x76531, 0x76532, 0x765321, 0x7654, - 0x76541, 0x76542, 0x765421, 0x76543, 0x765431, 0x765432, 0x7654321, 0x8, - 0x81, 0x82, 0x821, 0x83, 0x831, 0x832, 0x8321, 0x84, 0x841, 0x842, 0x8421, - 0x843, 0x8431, 0x8432, 0x84321, 0x85, 0x851, 0x852, 0x8521, 0x853, 0x8531, - 0x8532, 0x85321, 0x854, 0x8541, 0x8542, 0x85421, 0x8543, 0x85431, 0x85432, - 0x854321, 0x86, 0x861, 0x862, 0x8621, 0x863, 0x8631, 0x8632, 0x86321, 0x864, - 0x8641, 0x8642, 0x86421, 0x8643, 0x86431, 0x86432, 0x864321, 0x865, 0x8651, - 0x8652, 0x86521, 0x8653, 0x86531, 0x86532, 0x865321, 0x8654, 0x86541, - 0x86542, 0x865421, 0x86543, 0x865431, 0x865432, 0x8654321, 0x87, 0x871, - 0x872, 0x8721, 0x873, 0x8731, 0x8732, 0x87321, 0x874, 0x8741, 0x8742, - 0x87421, 0x8743, 0x87431, 0x87432, 0x874321, 0x875, 0x8751, 0x8752, 0x87521, - 0x8753, 0x87531, 0x87532, 0x875321, 0x8754, 0x87541, 0x87542, 0x875421, - 0x87543, 0x875431, 0x875432, 0x8754321, 0x876, 0x8761, 0x8762, 0x87621, - 0x8763, 0x87631, 0x87632, 0x876321, 0x8764, 0x87641, 0x87642, 0x876421, - 0x87643, 0x876431, 0x876432, 0x8764321, 0x8765, 0x87651, 0x87652, 0x876521, - 0x87653, 0x876531, 0x876532, 0x8765321, 0x87654, 0x876541, 0x876542, - 0x8765421, 0x876543, 0x8765431, 0x8765432, 0x87654321 - }; - /** - * ** the python code that generated bitlist - * def bits2int(val): - * arr=0 - * for shift in range(8,0,-1): - * if val & 0x80: - * arr = (arr << 4) | shift - * val = val << 1 - * return arr - * - * def int_table(): - * tbl = [ hex(bits2int(val)).strip('L') for val in range(256) ] - * return ','.join(tbl) - * **** - */ - - // hmmm, what about an iterator that finds zeros though, - // or a reverse iterator... should they be separate classes - // for efficiency, or have a common root interface? (or - // maybe both? could ask for a SetBitsIterator, etc... - - private final long[] arr; - private final int words; - private final int from; - private int i = -1; - private long word; - private int wordShift; - private int indexArray; - private int curDocId = -1; - - public SlicedIterator(SlicedOpenBitSet obs) { - this.arr = obs.bits; - this.words = obs.wlen; - this.from = obs.from; - } - - // 64 bit shifts - private void shift() { - if ((int) word == 0) { - wordShift += 32; - word = word >>> 32; - } - if ((word & 0x0000FFFF) == 0) { - wordShift += 16; - word >>>= 16; - } - if ((word & 0x000000FF) == 0) { - wordShift += 8; - word >>>= 8; - } - indexArray = bitlist[(int) word & 0xff]; - } - - /** - * ** alternate shift implementations - * // 32 bit shifts, but a long shift needed at the end - * private void shift2() { - * int y = (int)word; - * if (y==0) {wordShift +=32; y = (int)(word >>>32); } - * if ((y & 0x0000FFFF) == 0) { wordShift +=16; y>>>=16; } - * if ((y & 0x000000FF) == 0) { wordShift +=8; y>>>=8; } - * indexArray = bitlist[y & 0xff]; - * word >>>= (wordShift +1); - * } - * - * private void shift3() { - * int lower = (int)word; - * int lowByte = lower & 0xff; - * if (lowByte != 0) { - * indexArray=bitlist[lowByte]; - * return; - * } - * shift(); - * } - * **** - */ - - @Override - public int nextDoc() { - if (indexArray == 0) { - if (word != 0) { - word >>>= 8; - wordShift += 8; - } - - while (word == 0) { - if (++i >= words) { - return curDocId = NO_MORE_DOCS; - } - word = arr[from + i]; - wordShift = -1; // loop invariant code motion should move this - } - - // after the first time, should I go with a linear search, or - // stick with the binary search in shift? - shift(); - } - - int bitIndex = (indexArray & 0x0f) + wordShift; - indexArray >>>= 4; - // should i<<6 be cached as a separate variable? - // it would only save one cycle in the best circumstances. - return curDocId = (i << 6) + bitIndex; - } - - @Override - public int advance(int target) { - indexArray = 0; - i = target >> 6; - if (i >= words) { - word = 0; // setup so next() will also return -1 - return curDocId = NO_MORE_DOCS; - } - wordShift = target & 0x3f; - word = arr[from + i] >>> wordShift; - if (word != 0) { - wordShift--; // compensate for 1 based arrIndex - } else { - while (word == 0) { - if (++i >= words) { - return curDocId = NO_MORE_DOCS; - } - word = arr[from + i]; - } - wordShift = -1; - } - - shift(); - - int bitIndex = (indexArray & 0x0f) + wordShift; - indexArray >>>= 4; - // should i<<6 be cached as a separate variable? - // it would only save one cycle in the best circumstances. - return curDocId = (i << 6) + bitIndex; - } - - @Override - public int docID() { - return curDocId; - } - - } -} \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/TermFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/TermFilter.java index 6a5b32f2eaaef..79ca710d5571d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/TermFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/TermFilter.java @@ -24,7 +24,7 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import java.io.IOException; @@ -46,15 +46,15 @@ public Term getTerm() { } @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { - OpenBitSet result = null; + FixedBitSet result = null; TermDocs td = reader.termDocs(); try { td.seek(term); if (td.next()) { - result = new OpenBitSet(reader.maxDoc()); - result.fastSet(td.doc()); + result = new FixedBitSet(reader.maxDoc()); + result.set(td.doc()); while (td.next()) { - result.fastSet(td.doc()); + result.set(td.doc()); } } } finally { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java index e490e869bd17b..5662a72f6f76b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/search/XBooleanFilter.java @@ -20,11 +20,14 @@ package org.elasticsearch.common.lucene.search; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.*; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.FilterClause; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.OpenBitSetDISI; -import org.elasticsearch.common.lucene.docset.DocSet; -import org.elasticsearch.common.lucene.docset.OpenBitDocSet; +import org.elasticsearch.common.lucene.docset.DocSets; import java.io.IOException; import java.util.ArrayList; @@ -70,22 +73,15 @@ public List getNotFilters() { */ @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { - OpenBitSetDISI res = null; + FixedBitSet res = null; if (shouldFilters != null) { for (int i = 0; i < shouldFilters.size(); i++) { if (res == null) { - res = new OpenBitSetDISI(getDISI(shouldFilters, i, reader), reader.maxDoc()); + res = DocSets.createFixedBitSet(getDISI(shouldFilters, i, reader), reader.maxDoc()); } else { DocIdSet dis = shouldFilters.get(i).getDocIdSet(reader); - if (dis instanceof OpenBitSet) { - // optimized case for OpenBitSets - res.or((OpenBitSet) dis); - } else if (dis instanceof OpenBitDocSet) { - res.or(((OpenBitDocSet) dis).set()); - } else { - res.inPlaceOr(getDISI(shouldFilters, i, reader)); - } + DocSets.or(res, dis); } } } @@ -93,18 +89,11 @@ public DocIdSet getDocIdSet(IndexReader reader) throws IOException { if (notFilters != null) { for (int i = 0; i < notFilters.size(); i++) { if (res == null) { - res = new OpenBitSetDISI(getDISI(notFilters, i, reader), reader.maxDoc()); + res = DocSets.createFixedBitSet(getDISI(notFilters, i, reader), reader.maxDoc()); res.flip(0, reader.maxDoc()); // NOTE: may set bits on deleted docs } else { DocIdSet dis = notFilters.get(i).getDocIdSet(reader); - if (dis instanceof OpenBitSet) { - // optimized case for OpenBitSets - res.andNot((OpenBitSet) dis); - } else if (dis instanceof OpenBitDocSet) { - res.andNot(((OpenBitDocSet) dis).set()); - } else { - res.inPlaceNot(getDISI(notFilters, i, reader)); - } + DocSets.andNot(res, dis); } } } @@ -112,25 +101,15 @@ public DocIdSet getDocIdSet(IndexReader reader) throws IOException { if (mustFilters != null) { for (int i = 0; i < mustFilters.size(); i++) { if (res == null) { - res = new OpenBitSetDISI(getDISI(mustFilters, i, reader), reader.maxDoc()); + res = DocSets.createFixedBitSet(getDISI(mustFilters, i, reader), reader.maxDoc()); } else { DocIdSet dis = mustFilters.get(i).getDocIdSet(reader); - if (dis instanceof OpenBitSet) { - // optimized case for OpenBitSets - res.and((OpenBitSet) dis); - } else if (dis instanceof OpenBitDocSet) { - res.and(((OpenBitDocSet) dis).set()); - } else { - res.inPlaceAnd(getDISI(mustFilters, i, reader)); - } + DocSets.and(res, dis); } } } - if (res != null) - return new OpenBitDocSet(res); - - return DocSet.EMPTY_DOC_SET; + return res; } /** diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java index 8c70b1c04b7c3..f143c92956e21 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/lucene/uid/UidField.java @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.AbstractField; import org.apache.lucene.document.Field; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; @@ -135,10 +136,14 @@ public UidField(String name, String uid, long version) { super(name, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO); this.uid = uid; this.version = version; - this.omitTermFreqAndPositions = false; + this.indexOptions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; this.tokenStream = new UidPayloadTokenStream(this); } + @Override public void setIndexOptions(FieldInfo.IndexOptions indexOptions) { + // never allow to set this, since we want payload! + } + @Override public void setOmitTermFreqAndPositions(boolean omitTermFreqAndPositions) { // never allow to set this, since we want payload! } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java index 453ddf1452838..67bfc56c538d1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/analysis/SynonymTokenFilterFactory.java @@ -19,22 +19,30 @@ package org.elasticsearch.index.analysis; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.ReusableAnalyzerBase; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.WhitespaceTokenizer; import org.apache.lucene.analysis.synonym.SynonymFilter; import org.apache.lucene.analysis.synonym.SynonymMap; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.util.CharsRef; import org.elasticsearch.ElasticSearchIllegalArgumentException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.assistedinject.Assisted; -import org.elasticsearch.common.io.FastStringReader; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.indices.analysis.IndicesAnalysisService; +import java.io.BufferedReader; import java.io.IOException; +import java.io.LineNumberReader; +import java.io.Reader; +import java.text.ParseException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -43,6 +51,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { private final SynonymMap synonymMap; + private final boolean ignoreCase; @Inject public SynonymTokenFilterFactory(Index index, @IndexSettings Settings indexSettings, Environment env, IndicesAnalysisService indicesAnalysisService, Map tokenizerFactories, @Assisted String name, @Assisted Settings settings) { @@ -52,7 +61,7 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { if (rules == null) { throw new ElasticSearchIllegalArgumentException("synonym requires either `synonyms` or `synonyms_path` to be configured"); } - boolean ignoreCase = settings.getAsBoolean("ignore_case", false); + this.ignoreCase = settings.getAsBoolean("ignore_case", false); boolean expand = settings.getAsBoolean("expand", true); String tokenizerName = settings.get("tokenizer", "whitespace"); @@ -64,85 +73,183 @@ public class SynonymTokenFilterFactory extends AbstractTokenFilterFactory { if (tokenizerFactoryFactory == null) { throw new ElasticSearchIllegalArgumentException("failed to fine tokenizer [" + tokenizerName + "] for synonym token filter"); } - TokenizerFactory tokenizerFactory = tokenizerFactoryFactory.create(tokenizerName, settings); - synonymMap = new SynonymMap(ignoreCase); - parseRules(rules, synonymMap, "=>", ",", expand, tokenizerFactory); + final TokenizerFactory tokenizerFactory = tokenizerFactoryFactory.create(tokenizerName, settings); + + Analyzer analyzer = new ReusableAnalyzerBase() { + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer tokenizer = tokenizerFactory == null ? new WhitespaceTokenizer(Lucene.ANALYZER_VERSION, reader) : tokenizerFactory.create(reader); + TokenStream stream = ignoreCase ? new LowerCaseFilter(Lucene.ANALYZER_VERSION, tokenizer) : tokenizer; + return new TokenStreamComponents(tokenizer, stream); + } + }; + + CustomSynonymParser parser = new CustomSynonymParser(true, expand, analyzer); + try { + synonymMap = parser.build(); + } catch (IOException e) { + throw new ElasticSearchIllegalArgumentException("failed to build synonyms", e); + } } @Override public TokenStream create(TokenStream tokenStream) { - return new SynonymFilter(tokenStream, synonymMap); + // fst is null means no synonyms + return synonymMap.fst == null ? tokenStream : new SynonymFilter(tokenStream, synonymMap, ignoreCase); } - static void parseRules(List rules, SynonymMap map, String mappingSep, - String synSep, boolean expansion, TokenizerFactory tokFactory) { -// int count = 0; - for (String rule : rules) { - // To use regexes, we need an expression that specifies an odd number of chars. - // This can't really be done with string.split(), and since we need to - // do unescaping at some point anyway, we wouldn't be saving any effort - // by using regexes. - - List mapping = Strings.splitSmart(rule, mappingSep, false); - - List> source; - List> target; - - if (mapping.size() > 2) { - throw new RuntimeException("Invalid Synonym Rule:" + rule); - } else if (mapping.size() == 2) { - source = getSynList(mapping.get(0), synSep, tokFactory); - target = getSynList(mapping.get(1), synSep, tokFactory); + /** + * Parser for the Solr synonyms format. + *

    + *
  1. Blank lines and lines starting with '#' are comments. + *
  2. Explicit mappings match any token sequence on the LHS of "=>" + * and replace with all alternatives on the RHS. These types of mappings + * ignore the expand parameter in the constructor. + * Example: + *
    i-pod, i pod => ipod
    + *
  3. Equivalent synonyms may be separated with commas and give + * no explicit mapping. In this case the mapping behavior will + * be taken from the expand parameter in the constructor. This allows + * the same synonym file to be used in different synonym handling strategies. + * Example: + *
    ipod, i-pod, i pod
    + * + *
  4. Multiple synonym mapping entries are merged. + * Example: + *
    + * foo => foo bar
    + * foo => baz

    + * is equivalent to

    + * foo => foo bar, baz + *
    + *
+ * + * @lucene.experimental + */ + public static class CustomSynonymParser extends SynonymMap.Builder { + private final boolean expand; + private final Analyzer analyzer; + + public CustomSynonymParser(boolean dedup, boolean expand, Analyzer analyzer) { + super(dedup); + this.expand = expand; + this.analyzer = analyzer; + } + + public void add(Reader in) throws IOException, ParseException { + LineNumberReader br = new LineNumberReader(in); + try { + addInternal(br); + } catch (IllegalArgumentException e) { + ParseException ex = new ParseException("Invalid synonym rule at line " + br.getLineNumber(), 0); + ex.initCause(e); + throw ex; + } finally { + br.close(); + } + } + + public void addLine(String line) throws IOException { + if (line.length() == 0 || line.charAt(0) == '#') { + return; + } + + CharsRef inputs[]; + CharsRef outputs[]; + + // TODO: we could process this more efficiently. + String sides[] = split(line, "=>"); + if (sides.length > 1) { // explicit mapping + if (sides.length != 2) { + throw new IllegalArgumentException("more than one explicit mapping specified on the same line"); + } + String inputStrings[] = split(sides[0], ","); + inputs = new CharsRef[inputStrings.length]; + for (int i = 0; i < inputs.length; i++) { + inputs[i] = analyze(analyzer, unescape(inputStrings[i]).trim(), new CharsRef()); + } + + String outputStrings[] = split(sides[1], ","); + outputs = new CharsRef[outputStrings.length]; + for (int i = 0; i < outputs.length; i++) { + outputs[i] = analyze(analyzer, unescape(outputStrings[i]).trim(), new CharsRef()); + } } else { - source = getSynList(mapping.get(0), synSep, tokFactory); - if (expansion) { - // expand to all arguments - target = source; + String inputStrings[] = split(line, ","); + inputs = new CharsRef[inputStrings.length]; + for (int i = 0; i < inputs.length; i++) { + inputs[i] = analyze(analyzer, unescape(inputStrings[i]).trim(), new CharsRef()); + } + if (expand) { + outputs = inputs; } else { - // reduce to first argument - target = new ArrayList>(1); - target.add(source.get(0)); + outputs = new CharsRef[1]; + outputs[0] = inputs[0]; } } - boolean includeOrig = false; - for (List fromToks : source) { -// count++; - for (List toToks : target) { - map.add(fromToks, - SynonymMap.makeTokens(toToks), - includeOrig, - true - ); + // currently we include the term itself in the map, + // and use includeOrig = false always. + // this is how the existing filter does it, but its actually a bug, + // especially if combined with ignoreCase = true + for (int i = 0; i < inputs.length; i++) { + for (int j = 0; j < outputs.length; j++) { + add(inputs[i], outputs[j], false); } } } - } - // a , b c , d e f => [[a],[b,c],[d,e,f]] - private static List> getSynList(String str, String separator, TokenizerFactory tokFactory) { - List strList = Strings.splitSmart(str, separator, false); - // now split on whitespace to get a list of token strings - List> synList = new ArrayList>(); - for (String toks : strList) { - List tokList = tokFactory == null ? - Strings.splitWS(toks, true) : splitByTokenizer(toks, tokFactory); - synList.add(tokList); + private void addInternal(BufferedReader in) throws IOException { + String line = null; + while ((line = in.readLine()) != null) { + addLine(line); + } } - return synList; - } - private static List splitByTokenizer(String source, TokenizerFactory tokFactory) { - TokenStream ts = tokFactory.create(new FastStringReader(source)); - List tokList = new ArrayList(); - try { - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - while (ts.incrementToken()) { - if (termAtt.length() > 0) - tokList.add(termAtt.toString()); + private static String[] split(String s, String separator) { + ArrayList list = new ArrayList(2); + StringBuilder sb = new StringBuilder(); + int pos = 0, end = s.length(); + while (pos < end) { + if (s.startsWith(separator, pos)) { + if (sb.length() > 0) { + list.add(sb.toString()); + sb = new StringBuilder(); + } + pos += separator.length(); + continue; + } + + char ch = s.charAt(pos++); + if (ch == '\\') { + sb.append(ch); + if (pos >= end) break; // ERROR, or let it go? + ch = s.charAt(pos++); + } + + sb.append(ch); } - } catch (IOException e) { - throw new RuntimeException(e); + + if (sb.length() > 0) { + list.add(sb.toString()); + } + + return list.toArray(new String[list.size()]); + } + + private String unescape(String s) { + if (s.indexOf("\\") >= 0) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < s.length(); i++) { + char ch = s.charAt(i); + if (ch == '\\' && i < s.length() - 1) { + sb.append(s.charAt(++i)); + } else { + sb.append(ch); + } + } + return sb.toString(); + } + return s; } - return tokList; } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java index 9e5cb71111f18..01087a0ba5b71 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractConcurrentMapFilterCache.java @@ -22,13 +22,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; -import org.elasticsearch.common.RamUsage; -import org.elasticsearch.common.lab.LongsLAB; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.lucene.search.NoCacheFilter; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; import org.elasticsearch.index.cache.filter.FilterCache; @@ -49,30 +45,9 @@ public abstract class AbstractConcurrentMapFilterCache extends AbstractIndexComp final ConcurrentMap>> cache; - final boolean labEnabled; - final ByteSizeValue labMaxAlloc; - final ByteSizeValue labChunkSize; - - final int labMaxAllocBytes; - final int labChunkSizeBytes; - protected AbstractConcurrentMapFilterCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); this.cache = buildCache(); - - // The LAB is stored per reader, so whole chunks will be cleared once reader is discarded. - // This means that with filter entry specific based eviction, like access time - // we might get into cases where the LAB is held by a puny filter and other filters have been released. - // This usually will not be that bad, compared to the GC benefit of using a LAB, but, that is why - // the soft filter cache is recommended. - this.labEnabled = componentSettings.getAsBoolean("lab", false); - // These values should not be too high, basically we want to cached the small readers and use the LAB for - // them, 1M docs on OpenBitSet is around 110kb. - this.labMaxAlloc = componentSettings.getAsBytesSize("lab.max_alloc", new ByteSizeValue(128, ByteSizeUnit.KB)); - this.labChunkSize = componentSettings.getAsBytesSize("lab.chunk_size", new ByteSizeValue(1, ByteSizeUnit.MB)); - - this.labMaxAllocBytes = (int) (labMaxAlloc.bytes() / RamUsage.NUM_BYTES_LONG); - this.labChunkSizeBytes = (int) (labChunkSize.bytes() / RamUsage.NUM_BYTES_LONG); } protected ConcurrentMap>> buildCache() { @@ -153,11 +128,7 @@ static class FilterCacheFilterWrapper extends Filter { @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { FilterCacheValue> cacheValue = cache.cache.get(reader.getCoreCacheKey()); if (cacheValue == null) { - LongsLAB longsLAB = null; - if (cache.labEnabled) { - longsLAB = new LongsLAB(cache.labChunkSizeBytes, cache.labMaxAllocBytes); - } - cacheValue = new FilterCacheValue>(cache.buildFilterMap(), longsLAB); + cacheValue = new FilterCacheValue>(cache.buildFilterMap()); FilterCacheValue> prev = cache.cache.putIfAbsent(reader.getCoreCacheKey(), cacheValue); if (prev != null) { cacheValue = prev; @@ -175,7 +146,7 @@ static class FilterCacheFilterWrapper extends Filter { return docSet; } DocIdSet docIdSet = filter.getDocIdSet(reader); - docSet = FilterCacheValue.cacheable(reader, cacheValue.longsLAB(), docIdSet); + docSet = FilterCacheValue.cacheable(reader, docIdSet); DocSet prev = cacheValue.value().putIfAbsent(key, docSet); if (prev != null) { docSet = prev; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java index bf79e93dde90a..793241b20545a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/AbstractWeightedFilterCache.java @@ -23,17 +23,13 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.common.RamUsage; import org.elasticsearch.common.concurrentlinkedhashmap.EvictionListener; import org.elasticsearch.common.concurrentlinkedhashmap.Weigher; -import org.elasticsearch.common.lab.LongsLAB; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.lucene.search.NoCacheFilter; import org.elasticsearch.common.metrics.CounterMetric; import org.elasticsearch.common.metrics.MeanMetric; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.Index; @@ -48,32 +44,11 @@ public abstract class AbstractWeightedFilterCache extends AbstractIndexComponent final ConcurrentMap seenReaders = ConcurrentCollections.newConcurrentMap(); final CounterMetric seenReadersCount = new CounterMetric(); - final boolean labEnabled; - final ByteSizeValue labMaxAlloc; - final ByteSizeValue labChunkSize; - - final int labMaxAllocBytes; - final int labChunkSizeBytes; - final CounterMetric evictionsMetric = new CounterMetric(); final MeanMetric totalMetric = new MeanMetric(); protected AbstractWeightedFilterCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); - - // The LAB is stored per reader, so whole chunks will be cleared once reader is discarded. - // This means that with filter entry specific based eviction, like access time - // we might get into cases where the LAB is held by a puny filter and other filters have been released. - // This usually will not be that bad, compared to the GC benefit of using a LAB, but, that is why - // the soft filter cache is recommended. - this.labEnabled = componentSettings.getAsBoolean("lab", false); - // These values should not be too high, basically we want to cached the small readers and use the LAB for - // them, 1M docs on OpenBitSet is around 110kb. - this.labMaxAlloc = componentSettings.getAsBytesSize("lab.max_alloc", new ByteSizeValue(128, ByteSizeUnit.KB)); - this.labChunkSize = componentSettings.getAsBytesSize("lab.chunk_size", new ByteSizeValue(1, ByteSizeUnit.MB)); - - this.labMaxAllocBytes = (int) (labMaxAlloc.bytes() / RamUsage.NUM_BYTES_LONG); - this.labChunkSizeBytes = (int) (labChunkSize.bytes() / RamUsage.NUM_BYTES_LONG); } protected abstract ConcurrentMap> cache(); @@ -176,13 +151,9 @@ static class FilterCacheFilterWrapper extends Filter { } } - LongsLAB longsLAB = null; - if (cache.labEnabled) { - longsLAB = new LongsLAB(cache.labChunkSizeBytes, cache.labMaxAllocBytes); - } DocIdSet docIdSet = filter.getDocIdSet(reader); - DocSet docSet = FilterCacheValue.cacheable(reader, longsLAB, docIdSet); - cacheValue = new FilterCacheValue(docSet, longsLAB); + DocSet docSet = FilterCacheValue.cacheable(reader, docIdSet); + cacheValue = new FilterCacheValue(docSet); FilterCacheValue previous = innerCache.putIfAbsent(cacheKey, cacheValue); if (previous == null) { cache.totalMetric.inc(cacheValue.value().sizeInBytes()); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/FilterCacheValue.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/FilterCacheValue.java index 204ebc3fb95b3..bb8f6875cb051 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/FilterCacheValue.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/filter/support/FilterCacheValue.java @@ -22,36 +22,25 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.OpenBitSet; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.lab.LongsLAB; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.lucene.docset.DocSets; -import org.elasticsearch.common.lucene.docset.OpenBitDocSet; -import org.elasticsearch.common.lucene.docset.SlicedOpenBitSet; import java.io.IOException; public class FilterCacheValue { private final T value; - private final LongsLAB longsLAB; - public FilterCacheValue(T value, LongsLAB longsLAB) { + public FilterCacheValue(T value) { this.value = value; - this.longsLAB = longsLAB; } public T value() { return value; } - public LongsLAB longsLAB() { - return longsLAB; - } - - public static DocSet cacheable(IndexReader reader, @Nullable LongsLAB longsLAB, DocIdSet set) throws IOException { + public static DocSet cacheable(IndexReader reader, DocIdSet set) throws IOException { if (set == null) { return DocSet.EMPTY_DOC_SET; } @@ -67,29 +56,6 @@ public static DocSet cacheable(IndexReader reader, @Nullable LongsLAB longsLAB, if (doc == DocIdSetIterator.NO_MORE_DOCS) { return DocSet.EMPTY_DOC_SET; } - - // we have a LAB, check if can be used... - if (longsLAB == null) { - return DocSets.cacheable(reader, set); - } - - int numOfWords = OpenBitSet.bits2words(reader.maxDoc()); - LongsLAB.Allocation allocation = longsLAB.allocateLongs(numOfWords); - if (allocation == null) { - return DocSets.cacheable(reader, set); - } - // we have an allocation, use it to create SlicedOpenBitSet - if (set instanceof OpenBitSet) { - return new SlicedOpenBitSet(allocation.getData(), allocation.getOffset(), (OpenBitSet) set); - } else if (set instanceof OpenBitDocSet) { - return new SlicedOpenBitSet(allocation.getData(), allocation.getOffset(), ((OpenBitDocSet) set).set()); - } else { - SlicedOpenBitSet slicedSet = new SlicedOpenBitSet(allocation.getData(), numOfWords, allocation.getOffset()); - slicedSet.fastSet(doc); // we already have an open iterator, so use it, and don't forget to set the initial one - while ((doc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - slicedSet.fastSet(doc); - } - return slicedSet; - } + return DocSets.cacheable(reader, set); } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 7adb8b0713c94..23403ee5d16b3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.apache.lucene.search.FuzzyQuery; @@ -206,9 +207,10 @@ protected String buildFullName(BuilderContext context) { protected float boost; - protected boolean omitNorms; + protected final boolean omitNorms; - protected boolean omitTermFreqAndPositions; + protected final boolean omitTermFreqAndPositions; + protected final FieldInfo.IndexOptions indexOptions; protected final NamedAnalyzer indexAnalyzer; @@ -225,6 +227,7 @@ protected AbstractFieldMapper(Names names, Field.Index index, Field.Store store, this.boost = boost; this.omitNorms = omitNorms; this.omitTermFreqAndPositions = omitTermFreqAndPositions; + this.indexOptions = omitTermFreqAndPositions ? FieldInfo.IndexOptions.DOCS_ONLY : FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; if (indexAnalyzer == null && !index.isAnalyzed()) { this.indexAnalyzer = Lucene.KEYWORD_ANALYZER; } else { @@ -298,7 +301,7 @@ protected AbstractFieldMapper(Names names, Field.Index index, Field.Store store, return; } field.setOmitNorms(omitNorms); - field.setOmitTermFreqAndPositions(omitTermFreqAndPositions); + field.setIndexOptions(indexOptions); if (!customBoost()) { field.setBoost(boost); } @@ -414,8 +417,6 @@ protected boolean customBoost() { if (!mergeContext.mergeFlags().simulate()) { // apply changeable values this.boost = fieldMergeWith.boost; - this.omitNorms = fieldMergeWith.omitNorms; - this.omitTermFreqAndPositions = fieldMergeWith.omitTermFreqAndPositions; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index b4ea6c0cefd9c..f7be4fc34e560 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.AbstractField; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.util.NumericUtils; @@ -227,7 +228,7 @@ public CustomNumericField(NumberFieldMapper mapper, byte[] value) { isIndexed = mapper.indexed(); isTokenized = mapper.indexed(); - omitTermFreqAndPositions = true; + indexOptions = FieldInfo.IndexOptions.DOCS_ONLY; omitNorms = mapper.omitNorms(); if (value != null) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java index 4b1f50235b807..7d904127699e2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java @@ -24,7 +24,7 @@ import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.common.Unicode; import org.elasticsearch.common.bloom.BloomFilter; @@ -58,7 +58,7 @@ public UidFilter(Collection types, List ids, BloomCache bloomCac // - We can use sorted int array DocIdSet to reserve memory compared to OpenBitSet in some cases @Override public DocIdSet getDocIdSet(IndexReader reader) throws IOException { BloomFilter filter = bloomCache.filter(reader, UidFieldMapper.NAME, true); - OpenBitSet set = null; + FixedBitSet set = null; TermDocs td = null; try { for (Term uid : uids) { @@ -72,9 +72,9 @@ public UidFilter(Collection types, List ids, BloomCache bloomCac td.seek(uid); while (td.next()) { if (set == null) { - set = new OpenBitSet(reader.maxDoc()); + set = new FixedBitSet(reader.maxDoc()); } - set.fastSet(td.doc()); + set.set(td.doc()); } } } finally { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/ChildCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/ChildCollector.java index 827132518c569..c4032b5f7727b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/ChildCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/ChildCollector.java @@ -22,7 +22,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Collector; import org.apache.lucene.search.Scorer; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.BytesWrap; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.index.cache.id.IdReaderTypeCache; @@ -43,14 +43,14 @@ public class ChildCollector extends Collector { private final Tuple[] readers; - private final Map parentDocs; + private final Map parentDocs; private IdReaderTypeCache typeCache; public ChildCollector(String parentType, SearchContext context) { this.parentType = parentType; this.context = context; - this.parentDocs = new HashMap(); + this.parentDocs = new HashMap(); // create a specific type map lookup for faster lookup operations per doc this.readers = new Tuple[context.searcher().subReaders().length]; @@ -60,7 +60,7 @@ public ChildCollector(String parentType, SearchContext context) { } } - public Map parentDocs() { + public Map parentDocs() { return this.parentDocs; } @@ -81,12 +81,12 @@ public Map parentDocs() { } int parentDocId = idReaderTypeCache.docById(parentId); if (parentDocId != -1 && !indexReader.isDeleted(parentDocId)) { - OpenBitSet docIdSet = parentDocs().get(indexReader.getCoreCacheKey()); + FixedBitSet docIdSet = parentDocs().get(indexReader.getCoreCacheKey()); if (docIdSet == null) { - docIdSet = new OpenBitSet(indexReader.maxDoc()); + docIdSet = new FixedBitSet(indexReader.maxDoc()); parentDocs.put(indexReader.getCoreCacheKey(), docIdSet); } - docIdSet.fastSet(parentDocId); + docIdSet.set(parentDocId); return; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/HasChildFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/HasChildFilter.java index c0c1c28224bbb..e9b04399a5891 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/HasChildFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/child/HasChildFilter.java @@ -24,7 +24,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.search.internal.ScopePhase; import org.elasticsearch.search.internal.SearchContext; @@ -46,7 +46,7 @@ public class HasChildFilter extends Filter implements ScopePhase.CollectorPhase private final SearchContext searchContext; - private Map parentDocs; + private Map parentDocs; public HasChildFilter(Query query, String scope, String childType, String parentType, SearchContext searchContext) { this.query = query; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java index 908a6fbe230b9..a8226c10148ef 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/geo/IndexedGeoBoundingBoxFilter.java @@ -22,8 +22,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.ElasticSearchIllegalArgumentException; +import org.elasticsearch.common.lucene.docset.DocSets; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import java.io.IOException; @@ -56,13 +57,13 @@ public LeftGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointFieldM latFilter = fieldMapper.latMapper().rangeFilter(bottomRight.lat, topLeft.lat, true, true); } - @Override public OpenBitSet getDocIdSet(IndexReader reader) throws IOException { - OpenBitSet main; + @Override public FixedBitSet getDocIdSet(IndexReader reader) throws IOException { + FixedBitSet main; DocIdSet set = lonFilter1.getDocIdSet(reader); if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { main = null; } else { - main = (OpenBitSet) set; + main = (FixedBitSet) set; } set = lonFilter2.getDocIdSet(reader); @@ -74,9 +75,9 @@ public LeftGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointFieldM } } else { if (main == null) { - main = (OpenBitSet) set; + main = (FixedBitSet) set; } else { - main.or((OpenBitSet) set); + main.or((FixedBitSet) set); } } @@ -84,7 +85,7 @@ public LeftGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointFieldM if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { return null; } - main.and((OpenBitSet) set); + DocSets.and(main, set); return main; } @@ -121,18 +122,18 @@ public RightGeoBoundingBoxFilter(Point topLeft, Point bottomRight, GeoPointField latFilter = fieldMapper.latMapper().rangeFilter(bottomRight.lat, topLeft.lat, true, true); } - @Override public OpenBitSet getDocIdSet(IndexReader reader) throws IOException { - OpenBitSet main; + @Override public FixedBitSet getDocIdSet(IndexReader reader) throws IOException { + FixedBitSet main; DocIdSet set = lonFilter.getDocIdSet(reader); if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { return null; } - main = (OpenBitSet) set; + main = (FixedBitSet) set; set = latFilter.getDocIdSet(reader); if (set == null || set == DocIdSet.EMPTY_DOCIDSET) { return null; } - main.and((OpenBitSet) set); + DocSets.and(main, set); return main; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/BlockJoinQuery.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/BlockJoinQuery.java index 4d0f4f0a50ad4..1fd4c10218e3f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/BlockJoinQuery.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/BlockJoinQuery.java @@ -23,8 +23,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.*; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.OpenBitSet; -import org.elasticsearch.common.lucene.docset.OpenBitDocSet; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.common.lucene.docset.FixedBitDocSet; import org.elasticsearch.common.lucene.search.NoopCollector; import java.io.IOException; @@ -38,7 +38,7 @@ * child documents must appear first, ending with the parent * document. At search time you provide a Filter * identifying the parents, however this Filter must provide - * an {@link org.apache.lucene.util.OpenBitSet} per sub-reader. + * an {@link org.apache.lucene.util.FixedBitSet} per sub-reader. * *

Once the block index is built, use this query to wrap * any sub-query matching only child docs and join matches in that @@ -177,10 +177,10 @@ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topSc return null; } // CHANGE: - if (parents instanceof OpenBitDocSet) { - parents = ((OpenBitDocSet) parents).set(); + if (parents instanceof FixedBitDocSet) { + parents = ((FixedBitDocSet) parents).set(); } - if (!(parents instanceof OpenBitSet)) { + if (!(parents instanceof FixedBitSet)) { throw new IllegalStateException("parentFilter must return OpenBitSet; got " + parents); } @@ -190,7 +190,7 @@ public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topSc childCollector.setScorer(childScorer); } - return new BlockJoinScorer(this, childScorer, (OpenBitSet) parents, firstChildDoc, scoreMode, childCollector); + return new BlockJoinScorer(this, childScorer, (FixedBitSet) parents, firstChildDoc, scoreMode, childCollector); } @Override @@ -208,7 +208,7 @@ public boolean scoresDocsOutOfOrder() { static class BlockJoinScorer extends Scorer { private final Scorer childScorer; - private final OpenBitSet parentBits; + private final FixedBitSet parentBits; private final ScoreMode scoreMode; private final Collector childCollector; private int parentDoc; @@ -219,7 +219,7 @@ static class BlockJoinScorer extends Scorer { private float[] pendingChildScores; private int childDocUpto; - public BlockJoinScorer(Weight weight, Scorer childScorer, OpenBitSet parentBits, int firstChildDoc, ScoreMode scoreMode, Collector childCollector) { + public BlockJoinScorer(Weight weight, Scorer childScorer, FixedBitSet parentBits, int firstChildDoc, ScoreMode scoreMode, Collector childCollector) { super(weight); //System.out.println("Q.init firstChildDoc=" + firstChildDoc); this.parentBits = parentBits; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NestedChildrenCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NestedChildrenCollector.java index 0ff820fa06e64..949c6c26b42a4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NestedChildrenCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NestedChildrenCollector.java @@ -22,10 +22,10 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Filter; import org.apache.lucene.search.Scorer; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.common.lucene.docset.DocSet; import org.elasticsearch.common.lucene.docset.DocSets; -import org.elasticsearch.common.lucene.docset.OpenBitDocSet; +import org.elasticsearch.common.lucene.docset.FixedBitDocSet; import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.FacetCollector; @@ -44,7 +44,7 @@ public class NestedChildrenCollector extends FacetCollector { private DocSet childDocs; - private OpenBitSet parentDocs; + private FixedBitSet parentDocs; private IndexReader currentReader; @@ -71,7 +71,7 @@ public NestedChildrenCollector(FacetCollector collector, Filter parentFilter, Fi collector.setNextReader(reader, docBase); currentReader = reader; childDocs = DocSets.convert(reader, childFilter.getDocIdSet(reader)); - parentDocs = ((OpenBitDocSet) parentFilter.getDocIdSet(reader)).set(); + parentDocs = ((FixedBitDocSet) parentFilter.getDocIdSet(reader)).set(); } @Override public boolean acceptsDocsOutOfOrder() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NonNestedDocsFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NonNestedDocsFilter.java index 93d7914576013..f9f3deab85c61 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NonNestedDocsFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/nested/NonNestedDocsFilter.java @@ -24,7 +24,7 @@ import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.Filter; import org.apache.lucene.search.PrefixFilter; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.index.mapper.internal.TypeFieldMapper; import java.io.IOException; @@ -46,9 +46,9 @@ private NonNestedDocsFilter() { if (docSet == null || docSet == DocIdSet.EMPTY_DOCIDSET) { // will almost never happen, and we need an OpenBitSet for the parent filter in // BlockJoinQuery, we cache it anyhow... - docSet = new OpenBitSet(reader.maxDoc()); + docSet = new FixedBitSet(reader.maxDoc()); } - ((OpenBitSet) docSet).flip(0, reader.maxDoc()); + ((FixedBitSet) docSet).flip(0, reader.maxDoc()); return docSet; } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSetTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSetTests.java deleted file mode 100644 index ef2bcbd7234a6..0000000000000 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/common/lucene/docset/SlicedOpenBitSetTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.common.lucene.docset; - -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.OpenBitSet; -import org.testng.annotations.Test; - -import java.io.IOException; - -import static org.hamcrest.MatcherAssert.*; -import static org.hamcrest.Matchers.*; - -@Test -public class SlicedOpenBitSetTests { - - @Test public void simpleTests() throws IOException { - int numberOfBits = 500; - SlicedOpenBitSet bitSet = new SlicedOpenBitSet(new long[OpenBitSet.bits2words(numberOfBits) + 100], OpenBitSet.bits2words(numberOfBits), 100); - - bitSet.fastSet(100); - assertThat(bitSet.fastGet(100), equalTo(true)); - - DocIdSetIterator iterator = bitSet.iterator(); - assertThat(iterator.nextDoc(), equalTo(100)); - assertThat(iterator.nextDoc(), equalTo(DocIdSetIterator.NO_MORE_DOCS)); - } - - @Test public void testCopy() throws IOException { - int numberOfBits = 500; - OpenBitSet bitSet = new OpenBitSet(numberOfBits); - bitSet.set(100); - - SlicedOpenBitSet sBitSet = new SlicedOpenBitSet(new long[OpenBitSet.bits2words(numberOfBits) + 33], 33, bitSet); - assertThat(sBitSet.get(100), equalTo(true)); - } -} \ No newline at end of file diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java index 90eacf7b06189..16b0cb7a8eeb4 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/deps/lucene/SimpleLuceneTests.java @@ -31,7 +31,14 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; -import org.apache.lucene.search.*; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.NumericUtils; @@ -43,7 +50,6 @@ import java.util.ArrayList; import java.util.IdentityHashMap; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.lucene.DocumentBuilder.*; import static org.hamcrest.MatcherAssert.*; @@ -140,48 +146,6 @@ public class SimpleLuceneTests { indexWriter.close(); } - @Test public void testCollectorOrdering() throws Exception { - Directory dir = new RAMDirectory(); - IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER)); - for (int i = 0; i < 5000; i++) { - indexWriter.addDocument(doc() - .add(field("_id", Integer.toString(i))).build()); - if ((i % 131) == 0) { - indexWriter.commit(); - } - } - IndexReader reader = IndexReader.open(indexWriter, true); - IndexSearcher searcher = new IndexSearcher(reader); - - for (int i = 0; i < 5000; i++) { - final int index = i; - final AtomicInteger docId = new AtomicInteger(); - searcher.search(new MatchAllDocsQuery(), new Collector() { - int counter = 0; - int docBase = 0; - - @Override public void setScorer(Scorer scorer) throws IOException { - } - - @Override public void collect(int doc) throws IOException { - if (counter++ == index) { - docId.set(docBase + doc); - } - } - - @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { - this.docBase = docBase; - } - - @Override public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - Document doc = searcher.doc(docId.get()); - assertThat(doc.get("_id"), equalTo(Integer.toString(i))); - } - } - @Test public void testBoost() throws Exception { Directory dir = new RAMDirectory(); IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.VERSION, Lucene.STANDARD_ANALYZER)); diff --git a/plugins/analysis/icu/build.gradle b/plugins/analysis/icu/build.gradle index 44a92a9a664c7..70cac654c6ab1 100644 --- a/plugins/analysis/icu/build.gradle +++ b/plugins/analysis/icu/build.gradle @@ -35,8 +35,8 @@ dependencies { compile('org.apache.lucene:lucene-icu4j:3.2.0') { transitive = false } distLib('org.apache.lucene:lucene-icu4j:3.2.0') { transitive = false } - compile('org.apache.lucene:lucene-icu:3.3.0') { transitive = false } - distLib('org.apache.lucene:lucene-icu:3.3.0') { transitive = false } + compile('org.apache.lucene:lucene-icu:3.4.0') { transitive = false } + distLib('org.apache.lucene:lucene-icu:3.4.0') { transitive = false } } task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << { @@ -131,4 +131,4 @@ uploadArchives { eclipseClasspath { defaultOutputDir = file('build/eclipse-build') -} \ No newline at end of file +} From 639515c240a5a26a597d5a3bd60c5a9cd727763e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Sep 2011 12:21:12 +0300 Subject: [PATCH 24/96] Optimizing inactive (indexing wise) shard to only happen when there are no ongoing merges, closes #1336. --- .../indices/memory/IndexingMemoryBufferController.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java index 57889bf870294..2c62cb6814267 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java @@ -141,7 +141,8 @@ class ShardsIndicesStatusChecker implements Runnable { } // inactive? if (!status.inactive) { - if ((time - status.time) > inactiveTime.millis()) { + // mark it as inactive only if enough time has passed and there are no ongoing merges going on... + if ((time - status.time) > inactiveTime.millis() && ((InternalIndexShard) indexShard).mergeScheduler().stats().current() == 0) { try { ((InternalIndexShard) indexShard).engine().updateIndexingBufferSize(Engine.INACTIVE_SHARD_INDEXING_BUFFER); } catch (EngineClosedException e) { @@ -154,14 +155,14 @@ class ShardsIndicesStatusChecker implements Runnable { // inactive for this amount of time, mark it status.inactive = true; activeInactiveStatusChanges = true; - logger.debug("marking shard [{}][{}] as inactive (inactive_time[{}]), setting size to [{}]", indexShard.shardId().index().name(), indexShard.shardId().id(), inactiveTime, Engine.INACTIVE_SHARD_INDEXING_BUFFER); + logger.debug("marking shard [{}][{}] as inactive (inactive_time[{}]) indexing wise, setting size to [{}]", indexShard.shardId().index().name(), indexShard.shardId().id(), inactiveTime, Engine.INACTIVE_SHARD_INDEXING_BUFFER); } } } else { if (status.inactive) { status.inactive = false; activeInactiveStatusChanges = true; - logger.debug("marking shard [{}][{}] as active", indexShard.shardId().index().name(), indexShard.shardId().id()); + logger.debug("marking shard [{}][{}] as active indexing wise", indexShard.shardId().index().name(), indexShard.shardId().id()); } status.time = -1; } @@ -170,7 +171,7 @@ class ShardsIndicesStatusChecker implements Runnable { } } if (activeInactiveStatusChanges) { - calcAndSetShardIndexingBuffer("shards became active/inactive"); + calcAndSetShardIndexingBuffer("shards became active/inactive (indexing wise)"); } } } From 28f56262bc3667dc7c990e927344565479525d24 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Sep 2011 14:01:45 +0300 Subject: [PATCH 25/96] use a simpler API call --- .../indices/memory/IndexingMemoryBufferController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java index 2c62cb6814267..d696e6eda891b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/memory/IndexingMemoryBufferController.java @@ -142,7 +142,7 @@ class ShardsIndicesStatusChecker implements Runnable { // inactive? if (!status.inactive) { // mark it as inactive only if enough time has passed and there are no ongoing merges going on... - if ((time - status.time) > inactiveTime.millis() && ((InternalIndexShard) indexShard).mergeScheduler().stats().current() == 0) { + if ((time - status.time) > inactiveTime.millis() && indexShard.mergeStats().current() == 0) { try { ((InternalIndexShard) indexShard).engine().updateIndexingBufferSize(Engine.INACTIVE_SHARD_INDEXING_BUFFER); } catch (EngineClosedException e) { From 25c6e8512d5fd7ed2138cad713f2714f52e881c7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Sep 2011 14:56:37 +0300 Subject: [PATCH 26/96] fix full flush when no changes happen in the index, so the updated trans id is not written --- .../index/engine/robin/RobinEngine.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index fd34ab6782547..cda00abf59451 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -833,7 +833,19 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { long translogId = translogIdGenerator.incrementAndGet(); translog.newTransientTranslog(translogId); indexWriter.commit(MapBuilder.newMapBuilder().put(Translog.TRANSLOG_ID_KEY, Long.toString(translogId)).map()); - translog.makeTransientCurrent(); + if (flush.force()) { + // if we force, we might not have committed, we need to check that its the same id + Map commitUserData = IndexReader.getCommitUserData(store.directory()); + long committedTranslogId = Long.parseLong(commitUserData.get(Translog.TRANSLOG_ID_KEY)); + if (committedTranslogId != translogId) { + // we did not commit anything, revert to the old translog + translog.revertTransient(); + } else { + translog.makeTransientCurrent(); + } + } else { + translog.makeTransientCurrent(); + } } catch (Exception e) { translog.revertTransient(); throw new FlushFailedEngineException(shardId, e); From a7e43005bb5c3d0d646cc9c3754137cbe118459e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 15 Sep 2011 17:07:04 +0300 Subject: [PATCH 27/96] Rest Delete API does not honor the `version_type` parameter, closes #1337. --- .../rest/action/delete/RestDeleteAction.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java index afef0ccdda01e..3583b56d64634 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/delete/RestDeleteAction.java @@ -29,7 +29,14 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.elasticsearch.rest.*; +import org.elasticsearch.index.VersionType; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestXContentBuilder; @@ -59,6 +66,7 @@ public class RestDeleteAction extends BaseRestHandler { deleteRequest.listenerThreaded(false); // we don't spawn, then fork if local deleteRequest.operationThreaded(true); + deleteRequest.versionType(VersionType.fromString(request.param("version_type"), deleteRequest.versionType())); String replicationType = request.param("replication"); if (replicationType != null) { From bdfa07934ef9c2d90c1e0ed84e49b389d5969aba Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 16 Sep 2011 18:39:23 +0300 Subject: [PATCH 28/96] refactor creation of lucene directory and simplify different directories implemenation strcture --- .../index/engine/SimpleEngineBenchmark.java | 333 ----------- .../index/store/SimpleStoreBenchmark.java | 295 ---------- .../common/io/FileSystemUtils.java | 6 +- .../index/store/DirectoryService.java | 35 ++ .../elasticsearch/index/store/IndexStore.java | 2 +- .../org/elasticsearch/index/store/Store.java | 521 ++++++++++++++++-- .../index/store/StoreModule.java | 3 +- .../{FsStore.java => FsDirectoryService.java} | 77 +-- .../store/fs/MmapFsDirectoryService.java | 47 ++ .../index/store/fs/MmapFsIndexStore.java | 6 +- .../index/store/fs/MmapFsStore.java | 85 --- .../index/store/fs/NioFsDirectoryService.java | 47 ++ .../index/store/fs/NioFsIndexStore.java | 6 +- .../index/store/fs/NioFsStore.java | 85 --- .../store/fs/SimpleFsDirectoryService.java | 47 ++ .../index/store/fs/SimpleFsIndexStore.java | 6 +- .../index/store/fs/SimpleFsStore.java | 85 --- ...e.java => ByteBufferDirectoryService.java} | 33 +- .../store/memory/ByteBufferIndexStore.java | 6 +- ...RamStore.java => RamDirectoryService.java} | 32 +- .../index/store/ram/RamIndexStore.java | 6 +- .../index/store/support/AbstractStore.java | 517 ----------------- .../indices/recovery/RecoveryTarget.java | 4 +- .../TransportNodesListShardStoreMetaData.java | 6 +- .../engine/AbstractSimpleEngineTests.java | 6 +- 25 files changed, 740 insertions(+), 1556 deletions(-) delete mode 100644 modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java delete mode 100644 modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java rename modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/{FsStore.java => FsDirectoryService.java} (55%) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java delete mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java delete mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java delete mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java rename modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/{ByteBufferStore.java => ByteBufferDirectoryService.java} (67%) rename modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/{RamStore.java => RamDirectoryService.java} (65%) delete mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java diff --git a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java b/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java deleted file mode 100644 index 91296cd0f7375..0000000000000 --- a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/engine/SimpleEngineBenchmark.java +++ /dev/null @@ -1,333 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.index.engine; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.LoadFirstFieldSelector; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.analysis.AnalysisService; -import org.elasticsearch.index.cache.bloom.none.NoneBloomCache; -import org.elasticsearch.index.deletionpolicy.KeepOnlyLastDeletionPolicy; -import org.elasticsearch.index.deletionpolicy.SnapshotDeletionPolicy; -import org.elasticsearch.index.engine.Engine; -import org.elasticsearch.index.engine.robin.RobinEngine; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.merge.policy.LogByteSizeMergePolicyProvider; -import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider; -import org.elasticsearch.index.settings.IndexSettingsService; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.similarity.SimilarityService; -import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.memory.ByteBufferStore; -import org.elasticsearch.index.translog.fs.FsTranslog; -import org.elasticsearch.threadpool.ThreadPool; - -import java.io.File; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.elasticsearch.common.lucene.DocumentBuilder.*; -import static org.elasticsearch.common.settings.ImmutableSettings.Builder.*; - -/** - * @author kimchy (Shay Banon) - */ -public class SimpleEngineBenchmark { - - private final Store store; - - private final Engine engine; - - - private final AtomicInteger idGenerator = new AtomicInteger(); - - private String[] contentItems = new String[]{"test1", "test2", "test3"}; - - private static byte[] TRANSLOG_PAYLOAD = new byte[12]; - - private volatile int lastRefreshedId = 0; - - - private boolean create = false; - - private int searcherIterations = 10; - - private Thread[] searcherThreads = new Thread[1]; - - private int writerIterations = 10; - - private Thread[] writerThreads = new Thread[1]; - - private TimeValue refreshSchedule = new TimeValue(1, TimeUnit.SECONDS); - - private TimeValue flushSchedule = new TimeValue(1, TimeUnit.MINUTES); - - - private CountDownLatch latch; - private CyclicBarrier barrier1; - private CyclicBarrier barrier2; - - - // scheduled thread pool for both refresh and flush operations - private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2); - - public SimpleEngineBenchmark(Store store, Engine engine) { - this.store = store; - this.engine = engine; - } - - public SimpleEngineBenchmark numberOfContentItems(int numberOfContentItems) { - contentItems = new String[numberOfContentItems]; - for (int i = 0; i < contentItems.length; i++) { - contentItems[i] = "content" + i; - } - return this; - } - - public SimpleEngineBenchmark searcherThreads(int numberOfSearcherThreads) { - searcherThreads = new Thread[numberOfSearcherThreads]; - return this; - } - - public SimpleEngineBenchmark searcherIterations(int searcherIterations) { - this.searcherIterations = searcherIterations; - return this; - } - - public SimpleEngineBenchmark writerThreads(int numberOfWriterThreads) { - writerThreads = new Thread[numberOfWriterThreads]; - return this; - } - - public SimpleEngineBenchmark writerIterations(int writerIterations) { - this.writerIterations = writerIterations; - return this; - } - - public SimpleEngineBenchmark refreshSchedule(TimeValue refreshSchedule) { - this.refreshSchedule = refreshSchedule; - return this; - } - - public SimpleEngineBenchmark flushSchedule(TimeValue flushSchedule) { - this.flushSchedule = flushSchedule; - return this; - } - - public SimpleEngineBenchmark create(boolean create) { - this.create = create; - return this; - } - - public SimpleEngineBenchmark build() { - for (int i = 0; i < searcherThreads.length; i++) { - searcherThreads[i] = new Thread(new SearcherThread(), "Searcher[" + i + "]"); - } - for (int i = 0; i < writerThreads.length; i++) { - writerThreads[i] = new Thread(new WriterThread(), "Writer[" + i + "]"); - } - - latch = new CountDownLatch(searcherThreads.length + writerThreads.length); - barrier1 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1); - barrier2 = new CyclicBarrier(searcherThreads.length + writerThreads.length + 1); - - // warmup by indexing all content items - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - for (String contentItem : contentItems) { - int id = idGenerator.incrementAndGet(); - String sId = Integer.toString(id); - Document doc = doc().add(field("_id", sId)) - .add(field("content", contentItem)).build(); - ParsedDocument pDoc = new ParsedDocument(sId, sId, "type", null, -1, -1, doc, Lucene.STANDARD_ANALYZER, TRANSLOG_PAYLOAD, false); - if (create) { - engine.create(new Engine.Create(null, new Term("_id", sId), pDoc)); - } else { - engine.index(new Engine.Index(null, new Term("_id", sId), pDoc)); - } - } - engine.refresh(new Engine.Refresh(true)); - stopWatch.stop(); - System.out.println("Warmup of [" + contentItems.length + "] content items, took " + stopWatch.totalTime()); - - return this; - } - - public void run() throws Exception { - for (Thread t : searcherThreads) { - t.start(); - } - for (Thread t : writerThreads) { - t.start(); - } - barrier1.await(); - - Refresher refresher = new Refresher(); - scheduledExecutorService.scheduleWithFixedDelay(refresher, refreshSchedule.millis(), refreshSchedule.millis(), TimeUnit.MILLISECONDS); - Flusher flusher = new Flusher(); - scheduledExecutorService.scheduleWithFixedDelay(flusher, flushSchedule.millis(), flushSchedule.millis(), TimeUnit.MILLISECONDS); - - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - barrier2.await(); - - latch.await(); - stopWatch.stop(); - - System.out.println("Summary"); - System.out.println(" -- Readers [" + searcherThreads.length + "] with [" + searcherIterations + "] iterations"); - System.out.println(" -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations"); - System.out.println(" -- Took: " + stopWatch.totalTime()); - System.out.println(" -- Refresh [" + refresher.id + "] took: " + refresher.stopWatch.totalTime()); - System.out.println(" -- Flush [" + flusher.id + "] took: " + flusher.stopWatch.totalTime()); - System.out.println(" -- Store size " + store.estimateSize()); - - scheduledExecutorService.shutdown(); - - engine.refresh(new Engine.Refresh(true)); - stopWatch = new StopWatch(); - stopWatch.start(); - Engine.Searcher searcher = engine.searcher(); - TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), idGenerator.get() + 1); - stopWatch.stop(); - System.out.println(" -- Indexed [" + idGenerator.get() + "] docs, found [" + topDocs.totalHits + "] hits, took " + stopWatch.totalTime()); - searcher.release(); - } - - private String content(long number) { - return contentItems[((int) (number % contentItems.length))]; - } - - private class Flusher implements Runnable { - StopWatch stopWatch = new StopWatch(); - private int id; - - @Override public void run() { - stopWatch.start("" + ++id); - engine.flush(new Engine.Flush()); - stopWatch.stop(); - } - } - - private class Refresher implements Runnable { - StopWatch stopWatch = new StopWatch(); - private int id; - - @Override public synchronized void run() { - stopWatch.start("" + ++id); - int lastId = idGenerator.get(); - engine.refresh(new Engine.Refresh(true)); - lastRefreshedId = lastId; - stopWatch.stop(); - } - } - - private class SearcherThread implements Runnable { - @Override public void run() { - try { - barrier1.await(); - barrier2.await(); - for (int i = 0; i < searcherIterations; i++) { - Engine.Searcher searcher = engine.searcher(); - TopDocs topDocs = searcher.searcher().search(new TermQuery(new Term("content", content(i))), 10); - // read one - searcher.searcher().doc(topDocs.scoreDocs[0].doc, new LoadFirstFieldSelector()); - searcher.release(); - } - } catch (Exception e) { - System.out.println("Searcher thread failed"); - e.printStackTrace(); - } finally { - latch.countDown(); - } - } - } - - private class WriterThread implements Runnable { - @Override public void run() { - try { - barrier1.await(); - barrier2.await(); - for (int i = 0; i < writerIterations; i++) { - int id = idGenerator.incrementAndGet(); - String sId = Integer.toString(id); - Document doc = doc().add(field("_id", sId)) - .add(field("content", content(id))).build(); - ParsedDocument pDoc = new ParsedDocument(sId, sId, "type", null, -1, -1, doc, Lucene.STANDARD_ANALYZER, TRANSLOG_PAYLOAD, false); - if (create) { - engine.create(new Engine.Create(null, new Term("_id", sId), pDoc)); - } else { - engine.index(new Engine.Index(null, new Term("_id", sId), pDoc)); - } - } - } catch (Exception e) { - System.out.println("Writer thread failed"); - e.printStackTrace(); - } finally { - latch.countDown(); - } - } - } - - public static void main(String[] args) throws Exception { - ShardId shardId = new ShardId(new Index("index"), 1); - Settings settings = EMPTY_SETTINGS; - -// Store store = new RamStore(shardId, settings); - Store store = new ByteBufferStore(shardId, settings, null, new ByteBufferCache(settings)); -// Store store = new NioFsStore(shardId, settings); - - store.deleteContent(); - - ThreadPool threadPool = new ThreadPool(); - SnapshotDeletionPolicy deletionPolicy = new SnapshotDeletionPolicy(new KeepOnlyLastDeletionPolicy(shardId, settings)); - Engine engine = new RobinEngine(shardId, settings, new ThreadPool(), new IndexSettingsService(shardId.index(), settings), store, deletionPolicy, new FsTranslog(shardId, EMPTY_SETTINGS, new File("work/fs-translog")), new LogByteSizeMergePolicyProvider(store, new IndexSettingsService(shardId.index(), EMPTY_SETTINGS)), - new ConcurrentMergeSchedulerProvider(shardId, settings), new AnalysisService(shardId.index()), new SimilarityService(shardId.index()), new NoneBloomCache(shardId.index())); - engine.start(); - - SimpleEngineBenchmark benchmark = new SimpleEngineBenchmark(store, engine) - .numberOfContentItems(1000) - .searcherThreads(50).searcherIterations(10000) - .writerThreads(10).writerIterations(10000) - .refreshSchedule(new TimeValue(1, TimeUnit.SECONDS)) - .flushSchedule(new TimeValue(1, TimeUnit.MINUTES)) - .create(false) - .build(); - - benchmark.run(); - - engine.close(); - store.close(); - threadPool.shutdown(); - } -} diff --git a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java b/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java deleted file mode 100644 index e34d379cb0a0a..0000000000000 --- a/modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.index.store; - -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.StopWatch; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.fs.*; -import org.elasticsearch.index.store.memory.ByteBufferStore; -import org.elasticsearch.index.store.ram.RamStore; - -import java.lang.management.ManagementFactory; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicLong; - -import static java.util.concurrent.TimeUnit.*; -import static org.elasticsearch.common.settings.ImmutableSettings.Builder.*; - -/** - * @author kimchy - */ -public class SimpleStoreBenchmark { - - private final AtomicLong dynamicFilesCounter = new AtomicLong(); - - private final Store store; - - private String[] staticFiles = new String[10]; - - private ByteSizeValue staticFileSize = new ByteSizeValue(5, ByteSizeUnit.MB); - - private ByteSizeValue dynamicFileSize = new ByteSizeValue(1, ByteSizeUnit.MB); - - - private int readerIterations = 10; - - private int writerIterations = 10; - - private Thread[] readerThreads = new Thread[1]; - - private Thread[] writerThreads = new Thread[1]; - - private CountDownLatch latch; - private CyclicBarrier barrier1; - private CyclicBarrier barrier2; - - public SimpleStoreBenchmark(Store store) throws Exception { - this.store = store; - } - - public SimpleStoreBenchmark numberStaticFiles(int numberStaticFiles) { - this.staticFiles = new String[numberStaticFiles]; - return this; - } - - public SimpleStoreBenchmark staticFileSize(ByteSizeValue staticFileSize) { - this.staticFileSize = staticFileSize; - return this; - } - - public SimpleStoreBenchmark dynamicFileSize(ByteSizeValue dynamicFileSize) { - this.dynamicFileSize = dynamicFileSize; - return this; - } - - public SimpleStoreBenchmark readerThreads(int readerThreads) { - this.readerThreads = new Thread[readerThreads]; - return this; - } - - public SimpleStoreBenchmark readerIterations(int readerIterations) { - this.readerIterations = readerIterations; - return this; - } - - public SimpleStoreBenchmark writerIterations(int writerIterations) { - this.writerIterations = writerIterations; - return this; - } - - public SimpleStoreBenchmark writerThreads(int writerThreads) { - this.writerThreads = new Thread[writerThreads]; - return this; - } - - public SimpleStoreBenchmark build() throws Exception { - System.out.println("Creating [" + staticFiles.length + "] static files with size [" + staticFileSize + "]"); - for (int i = 0; i < staticFiles.length; i++) { - staticFiles[i] = "static" + i; - IndexOutput io = store.directory().createOutput(staticFiles[i]); - for (long sizeCounter = 0; sizeCounter < staticFileSize.bytes(); sizeCounter++) { - io.writeByte((byte) 1); - } - io.close(); - } - System.out.println("Using [" + dynamicFileSize + "] size for dynamic files"); - - // warmp - StopWatch stopWatch = new StopWatch("warmup"); - stopWatch.start(); - for (String staticFile : staticFiles) { - IndexInput ii = store.directory().openInput(staticFile); - // do a full read - for (long counter = 0; counter < ii.length(); counter++) { - byte result = ii.readByte(); - if (result != 1) { - System.out.println("Failure, read wrong value [" + result + "]"); - } - } - // do a list of the files - store.directory().listAll(); - } - stopWatch.stop(); - System.out.println("Warmup Took: " + stopWatch.shortSummary()); - - for (int i = 0; i < readerThreads.length; i++) { - readerThreads[i] = new Thread(new ReaderThread(), "Reader[" + i + "]"); - } - for (int i = 0; i < writerThreads.length; i++) { - writerThreads[i] = new Thread(new WriterThread(), "Writer[" + i + "]"); - } - - latch = new CountDownLatch(readerThreads.length + writerThreads.length); - barrier1 = new CyclicBarrier(readerThreads.length + writerThreads.length + 1); - barrier2 = new CyclicBarrier(readerThreads.length + writerThreads.length + 1); - - return this; - } - - public void run() throws Exception { - for (int i = 0; i < 3; i++) { - System.gc(); - MILLISECONDS.sleep(100); - } - - long emptyUsed = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed(); - - System.out.println("Running:"); - System.out.println(" -- Readers [" + readerThreads.length + "] with [" + readerIterations + "] iterations"); - System.out.println(" -- Writers [" + writerThreads.length + "] with [" + writerIterations + "] iterations"); - for (Thread t : readerThreads) { - t.start(); - } - for (Thread t : writerThreads) { - t.start(); - } - barrier1.await(); - - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - barrier2.await(); - - latch.await(); - stopWatch.stop(); - - System.out.println("Took: " + stopWatch.shortSummary()); - - for (int i = 0; i < 3; i++) { - System.gc(); - MILLISECONDS.sleep(100); - } - long bytesTaken = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed() - emptyUsed; - System.out.println("Size of [" + staticFiles.length + "], each with size [" + staticFileSize + "], is " + new ByteSizeValue(bytesTaken, ByteSizeUnit.BYTES)); - } - - private class ReaderThread implements Runnable { - @Override public void run() { - try { - barrier1.await(); - barrier2.await(); - } catch (Exception e) { - e.printStackTrace(); - } - try { - for (int i = 0; i < readerIterations; i++) { - for (String staticFile : staticFiles) { - // do a list of the files - store.directory().listAll(); - - IndexInput ii = store.directory().openInput(staticFile); - // do a full read - for (long counter = 0; counter < ii.length(); counter++) { - byte result = ii.readByte(); - if (result != 1) { - System.out.println("Failure, read wrong value [" + result + "]"); - } - } - // do a list of the files - store.directory().listAll(); - - // do a seek and read some byes - ii.seek(ii.length() / 2); - ii.readByte(); - ii.readByte(); - - // do a list of the files - store.directory().listAll(); - } - } - } catch (Exception e) { - System.out.println("Reader Thread failed: " + e.getMessage()); - e.printStackTrace(); - } - latch.countDown(); - } - } - - private class WriterThread implements Runnable { - @Override public void run() { - try { - barrier1.await(); - barrier2.await(); - } catch (Exception e) { - e.printStackTrace(); - } - try { - for (int i = 0; i < writerIterations; i++) { - String dynamicFileName = "dynamic" + dynamicFilesCounter.incrementAndGet(); - IndexOutput io = store.directory().createOutput(dynamicFileName); - for (long sizeCounter = 0; sizeCounter < dynamicFileSize.bytes(); sizeCounter++) { - io.writeByte((byte) 1); - } - io.close(); - - store.directory().deleteFile(dynamicFileName); - } - } catch (Exception e) { - System.out.println("Writer thread failed: " + e.getMessage()); - e.printStackTrace(); - } - latch.countDown(); - } - } - - public static void main(String[] args) throws Exception { - Environment environment = new Environment(); - Settings settings = EMPTY_SETTINGS; - NodeEnvironment nodeEnvironment = new NodeEnvironment(settings, environment); - ByteBufferCache byteBufferCache = new ByteBufferCache(settings); - - ShardId shardId = new ShardId(new Index("index"), 1); - String type = args.length > 0 ? args[0] : "ram"; - Store store; - if (type.equalsIgnoreCase("ram")) { - store = new RamStore(shardId, settings, null); - } else if (type.equalsIgnoreCase("simple-fs")) { - store = new SimpleFsStore(shardId, settings, new SimpleFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache); - } else if (type.equalsIgnoreCase("mmap-fs")) { - store = new NioFsStore(shardId, settings, new NioFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache); - } else if (type.equalsIgnoreCase("nio-fs")) { - store = new MmapFsStore(shardId, settings, new MmapFsIndexStore(shardId.index(), settings, null, nodeEnvironment), byteBufferCache); - } else if (type.equalsIgnoreCase("memory")) { - store = new ByteBufferStore(shardId, settings, null, byteBufferCache); - } else { - throw new IllegalArgumentException("No type store [" + type + "]"); - } - System.out.println("Using Store [" + store + "]"); - store.deleteContent(); - SimpleStoreBenchmark simpleStoreBenchmark = new SimpleStoreBenchmark(store) - .numberStaticFiles(5).staticFileSize(new ByteSizeValue(5, ByteSizeUnit.MB)) - .dynamicFileSize(new ByteSizeValue(1, ByteSizeUnit.MB)) - .readerThreads(5).readerIterations(10) - .writerThreads(2).writerIterations(10) - .build(); - simpleStoreBenchmark.run(); - store.close(); - } -} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index 50eeb0c4f6d74..a5efd0e911386 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -124,6 +124,10 @@ public static boolean deleteRecursively(File root) { return deleteRecursively(root, true); } + private static boolean innerDeleteRecursively(File root) { + return deleteRecursively(root, true); + } + /** * Delete the supplied {@link java.io.File} - for directories, * recursively delete any nested directories or files as well. @@ -139,7 +143,7 @@ public static boolean deleteRecursively(File root, boolean deleteRoot) { File[] children = root.listFiles(); if (children != null) { for (File aChildren : children) { - deleteRecursively(aChildren); + innerDeleteRecursively(aChildren); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java new file mode 100644 index 0000000000000..f85ed364168f7 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store; + +import org.apache.lucene.store.Directory; + +import java.io.IOException; + +/** + */ +public interface DirectoryService { + + Directory build() throws IOException; + + void renameFile(Directory dir, String from, String to) throws IOException; + + void fullDelete(Directory dir) throws IOException; +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/IndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/IndexStore.java index ccf82c37452cb..27c3eedf26a75 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/IndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/IndexStore.java @@ -40,7 +40,7 @@ public interface IndexStore extends IndexComponent { /** * The shard store class that should be used for each shard. */ - Class shardStoreClass(); + Class shardDirectory(); /** * Returns the backing store total space. Return -1 if not available. diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java index 3977393b73fbc..04fbafa37b291 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java @@ -20,65 +20,516 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockFactory; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.Unicode; import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.lucene.Directories; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.shard.IndexShardComponent; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.AbstractIndexShardComponent; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.support.ForceSyncDirectory; +import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; import java.util.Map; +import java.util.zip.Adler32; +import java.util.zip.Checksum; /** - * @author kimchy (shay.banon) */ -public interface Store extends IndexShardComponent { +public class Store extends AbstractIndexShardComponent { - /** - * The Lucene {@link Directory} this store is using. - */ - Directory directory(); + static final String CHECKSUMS_PREFIX = "_checksums-"; - IndexOutput createOutputWithNoChecksum(String name) throws IOException; + public static final boolean isChecksum(String name) { + return name.startsWith(CHECKSUMS_PREFIX); + } - void writeChecksum(String name, String checksum) throws IOException; + private final IndexStore indexStore; - void writeChecksums(Map checksums) throws IOException; + private final DirectoryService directoryService; - StoreFileMetaData metaData(String name) throws IOException; + private final StoreDirectory directory; - ImmutableMap list() throws IOException; + private volatile ImmutableMap filesMetadata = ImmutableMap.of(); - /** - * Just deletes the content of the store. - */ - void deleteContent() throws IOException; + private volatile String[] files = Strings.EMPTY_ARRAY; - /** - * Renames, note, might not be atomic, and can fail "in the middle". - */ - void renameFile(String from, String to) throws IOException; + private final Object mutex = new Object(); - /** - * Deletes the store completely. For example, in FS ones, also deletes the parent - * directory. - */ - void fullDelete() throws IOException; + private final boolean sync; - StoreStats stats() throws IOException; + @Inject public Store(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, DirectoryService directoryService) throws IOException { + super(shardId, indexSettings); + this.indexStore = indexStore; + this.directoryService = directoryService; + this.sync = componentSettings.getAsBoolean("sync", true); // TODO we don't really need to fsync when using shared gateway... + this.directory = new StoreDirectory(directoryService.build()); + } - /** - * The estimated size this store is using. - */ - ByteSizeValue estimateSize() throws IOException; + public Directory directory() { + return directory; + } + + public ImmutableMap list() throws IOException { + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (String name : files) { + StoreFileMetaData md = metaData(name); + if (md != null) { + builder.put(md.name(), md); + } + } + return builder.build(); + } + + public StoreFileMetaData metaData(String name) throws IOException { + StoreFileMetaData md = filesMetadata.get(name); + if (md == null) { + return null; + } + // IndexOutput not closed, does not exists + if (md.lastModified() == -1 || md.length() == -1) { + return null; + } + return md; + } + + public void deleteContent() throws IOException { + String[] files = directory.listAll(); + IOException lastException = null; + for (String file : files) { + if (isChecksum(file)) { + try { + directory.deleteFileChecksum(file); + } catch (IOException e) { + lastException = e; + } + } else { + try { + directory.deleteFile(file); + } catch (FileNotFoundException e) { + // ignore + } catch (IOException e) { + lastException = e; + } + } + } + if (lastException != null) { + throw lastException; + } + } + + public void fullDelete() throws IOException { + deleteContent(); + directoryService.fullDelete(directory.delegate()); + } + + public StoreStats stats() throws IOException { + return new StoreStats(Directories.estimateSize(directory)); + } + + public ByteSizeValue estimateSize() throws IOException { + return new ByteSizeValue(Directories.estimateSize(directory)); + } + + public void renameFile(String from, String to) throws IOException { + directoryService.renameFile(directory.delegate(), from, to); + synchronized (mutex) { + StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one + StoreFileMetaData toMetaData = new StoreFileMetaData(to, fromMetaData.length(), fromMetaData.lastModified(), fromMetaData.checksum()); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(from).put(to, toMetaData).immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + } + + public static Map readChecksums(Directory dir) throws IOException { + long lastFound = -1; + for (String name : dir.listAll()) { + if (!isChecksum(name)) { + continue; + } + long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); + if (current > lastFound) { + lastFound = current; + } + } + if (lastFound == -1) { + return ImmutableMap.of(); + } + IndexInput indexInput = dir.openInput(CHECKSUMS_PREFIX + lastFound); + try { + indexInput.readInt(); // version + return indexInput.readStringStringMap(); + } catch (Exception e) { + // failed to load checksums, ignore and return an empty map + return new HashMap(); + } finally { + indexInput.close(); + } + } + + public void writeChecksums() throws IOException { + writeChecksums(directory); + } + + private void writeChecksums(StoreDirectory dir) throws IOException { + String checksumName = CHECKSUMS_PREFIX + System.currentTimeMillis(); + ImmutableMap files = list(); + synchronized (mutex) { + Map checksums = new HashMap(); + for (StoreFileMetaData metaData : files.values()) { + if (metaData.checksum() != null) { + checksums.put(metaData.name(), metaData.checksum()); + } + } + IndexOutput output = dir.createOutput(checksumName, false); + output.writeInt(0); // version + output.writeStringStringMap(checksums); + output.close(); + } + for (StoreFileMetaData metaData : files.values()) { + if (metaData.name().startsWith(CHECKSUMS_PREFIX) && !checksumName.equals(metaData.name())) { + try { + dir.deleteFileChecksum(metaData.name()); + } catch (Exception e) { + // ignore + } + } + } + } /** - * The store can suggest the best setting for compound file the - * {@link org.apache.lucene.index.MergePolicy} will use. + * Returns true by default. */ - boolean suggestUseCompoundFile(); + public boolean suggestUseCompoundFile() { + return false; + } + + public void close() throws IOException { + directory.close(); + } + + public IndexOutput createOutputWithNoChecksum(String name) throws IOException { + return directory.createOutput(name, false); + } + + public void writeChecksum(String name, String checksum) throws IOException { + // update the metadata to include the checksum and write a new checksums file + synchronized (mutex) { + StoreFileMetaData metaData = filesMetadata.get(name); + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), checksum); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); + writeChecksums(); + } + } + + public void writeChecksums(Map checksums) throws IOException { + // update the metadata to include the checksum and write a new checksums file + synchronized (mutex) { + for (Map.Entry entry : checksums.entrySet()) { + StoreFileMetaData metaData = filesMetadata.get(entry.getKey()); + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), entry.getValue()); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(entry.getKey(), metaData).immutableMap(); + } + writeChecksums(); + } + } /** - * Close the store. + * The idea of the store directory is to cache file level meta data, as well as md5 of it */ - void close() throws IOException; + protected class StoreDirectory extends Directory implements ForceSyncDirectory { + + private final Directory delegate; + + StoreDirectory(Directory delegate) throws IOException { + this.delegate = delegate; + synchronized (mutex) { + Map checksums = readChecksums(delegate); + MapBuilder builder = MapBuilder.newMapBuilder(); + for (String file : delegate.listAll()) { + // BACKWARD CKS SUPPORT + if (file.endsWith(".cks")) { // ignore checksum files here + continue; + } + String checksum = checksums.get(file); + + // BACKWARD CKS SUPPORT + if (checksum == null) { + if (delegate.fileExists(file + ".cks")) { + IndexInput indexInput = delegate.openInput(file + ".cks"); + try { + if (indexInput.length() > 0) { + byte[] checksumBytes = new byte[(int) indexInput.length()]; + indexInput.readBytes(checksumBytes, 0, checksumBytes.length, false); + checksum = Unicode.fromBytes(checksumBytes); + } + } finally { + indexInput.close(); + } + } + } + builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), checksum)); + } + filesMetadata = builder.immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + } + + public Directory delegate() { + return delegate; + } + + @Override public String[] listAll() throws IOException { + return files; + } + + @Override public boolean fileExists(String name) throws IOException { + return filesMetadata.containsKey(name); + } + + @Override public long fileModified(String name) throws IOException { + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData == null) { + throw new FileNotFoundException(name); + } + // not set yet (IndexOutput not closed) + if (metaData.lastModified() != -1) { + return metaData.lastModified(); + } + return delegate.fileModified(name); + } + + @Override public void touchFile(String name) throws IOException { + delegate.touchFile(name); + synchronized (mutex) { + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData != null) { + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), delegate.fileModified(name), metaData.checksum()); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); + } + } + } + + public void deleteFileChecksum(String name) throws IOException { + try { + delegate.deleteFile(name); + } catch (IOException e) { + if (delegate.fileExists(name)) { + throw e; + } + } + synchronized (mutex) { + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(name).immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + } + + @Override public void deleteFile(String name) throws IOException { + // we don't allow to delete the checksums files, only using the deleteChecksum method + if (isChecksum(name)) { + return; + } + try { + delegate.deleteFile(name); + } catch (IOException e) { + if (delegate.fileExists(name)) { + throw e; + } + } + synchronized (mutex) { + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(name).immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + } + + @Override public long fileLength(String name) throws IOException { + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData == null) { + throw new FileNotFoundException(name); + } + // not set yet (IndexOutput not closed) + if (metaData.length() != -1) { + return metaData.length(); + } + return delegate.fileLength(name); + } + + @Override public IndexOutput createOutput(String name) throws IOException { + return createOutput(name, true); + } + + public IndexOutput createOutput(String name, boolean computeChecksum) throws IOException { + IndexOutput out = delegate.createOutput(name); + synchronized (mutex) { + StoreFileMetaData metaData = new StoreFileMetaData(name, -1, -1, null); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + return new StoreIndexOutput(out, name, computeChecksum); + } + + @Override public IndexInput openInput(String name) throws IOException { + return delegate.openInput(name); + } + + @Override public void close() throws IOException { + delegate.close(); + synchronized (mutex) { + filesMetadata = ImmutableMap.of(); + files = Strings.EMPTY_ARRAY; + } + } + + @Override public Lock makeLock(String name) { + return delegate.makeLock(name); + } + + @Override public IndexInput openInput(String name, int bufferSize) throws IOException { + return delegate.openInput(name, bufferSize); + } + + @Override public void clearLock(String name) throws IOException { + delegate.clearLock(name); + } + + @Override public void setLockFactory(LockFactory lockFactory) throws IOException { + delegate.setLockFactory(lockFactory); + } + + @Override public LockFactory getLockFactory() { + return delegate.getLockFactory(); + } + + @Override public String getLockID() { + return delegate.getLockID(); + } + + @Override public void sync(Collection names) throws IOException { + if (sync) { + delegate.sync(names); + } + for (String name : names) { + // write the checksums file when we sync on the segments file (committed) + if (!name.equals("segments.gen") && name.startsWith("segments")) { + writeChecksums(); + break; + } + } + } + + @Override public void sync(String name) throws IOException { + if (sync) { + delegate.sync(name); + } + // write the checksums file when we sync on the segments file (committed) + if (!name.equals("segments.gen") && name.startsWith("segments")) { + writeChecksums(); + } + } + + @Override public void forceSync(String name) throws IOException { + delegate.sync(name); + } + } + + class StoreIndexOutput extends IndexOutput { + + private final IndexOutput delegate; + + private final String name; + + private final Checksum digest; + + StoreIndexOutput(IndexOutput delegate, String name, boolean computeChecksum) { + this.delegate = delegate; + this.name = name; + if (computeChecksum) { + if ("segments.gen".equals(name)) { + // no need to create checksum for segments.gen since its not snapshot to recovery + this.digest = null; + } else if (name.startsWith("segments")) { + // don't compute checksum for segments files, so pure Lucene can open this directory + // and since we, in any case, always recover the segments files + this.digest = null; + } else { +// this.digest = new CRC32(); + // adler is faster, and we compare on length as well, should be enough to check for difference + // between files + this.digest = new Adler32(); + } + } else { + this.digest = null; + } + } + + @Override public void close() throws IOException { + delegate.close(); + String checksum = null; + if (digest != null) { + checksum = Long.toString(digest.getValue(), Character.MAX_RADIX); + } + synchronized (mutex) { + StoreFileMetaData md = new StoreFileMetaData(name, directory.delegate().fileLength(name), directory.delegate().fileModified(name), checksum); + filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, md).immutableMap(); + files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + } + } + + @Override public void writeByte(byte b) throws IOException { + delegate.writeByte(b); + if (digest != null) { + digest.update(b); + } + } + + @Override public void writeBytes(byte[] b, int offset, int length) throws IOException { + delegate.writeBytes(b, offset, length); + if (digest != null) { + digest.update(b, offset, length); + } + } + + // don't override it, base class method simple reads from input and writes to this output +// @Override public void copyBytes(IndexInput input, long numBytes) throws IOException { +// delegate.copyBytes(input, numBytes); +// } + + @Override public void flush() throws IOException { + delegate.flush(); + } + + @Override public long getFilePointer() { + return delegate.getFilePointer(); + } + + @Override public void seek(long pos) throws IOException { + // seek might be called on files, which means that the checksum is not file checksum + // but a checksum of the bytes written to this stream, which is the same for each + // type of file in lucene + delegate.seek(pos); + } + + @Override public long length() throws IOException { + return delegate.length(); + } + + @Override public void setLength(long length) throws IOException { + delegate.setLength(length); + } + + @Override public void writeStringStringMap(Map map) throws IOException { + delegate.writeStringStringMap(map); + } + } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java index ec036805e130a..296f71529a7cb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreModule.java @@ -37,7 +37,8 @@ public StoreModule(Settings settings, IndexStore indexStore) { } @Override protected void configure() { - bind(Store.class).to(indexStore.shardStoreClass()).asEagerSingleton(); + bind(DirectoryService.class).to(indexStore.shardDirectory()).asEagerSingleton(); bind(StoreManagement.class).asEagerSingleton(); + bind(Store.class).asEagerSingleton(); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java similarity index 55% rename from modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsStore.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java index 8edc0fe0be001..0de110e3965cf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java @@ -25,17 +25,13 @@ import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.SimpleFSLockFactory; -import org.apache.lucene.store.bytebuffer.ByteBufferDirectory; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.collect.ImmutableSet; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.lucene.store.SwitchDirectory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.support.AbstractStore; import java.io.File; import java.io.FileNotFoundException; @@ -43,27 +39,30 @@ import java.io.InterruptedIOException; /** - * @author kimchy (shay.banon) */ -public abstract class FsStore extends AbstractStore { +public abstract class FsDirectoryService extends AbstractIndexShardComponent implements DirectoryService { - public static final boolean DEFAULT_SUGGEST_USE_COMPOUND_FILE = false; + protected final FsIndexStore indexStore; - public FsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { - super(shardId, indexSettings, indexStore); + public FsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings); + this.indexStore = (FsIndexStore) indexStore; } - @Override public void fullDelete() throws IOException { - FileSystemUtils.deleteRecursively(fsDirectory().getDirectory()); - // if we are the last ones, delete also the actual index - String[] list = fsDirectory().getDirectory().getParentFile().list(); - if (list == null || list.length == 0) { - FileSystemUtils.deleteRecursively(fsDirectory().getDirectory().getParentFile()); + protected LockFactory buildLockFactory() throws IOException { + String fsLock = componentSettings.get("fs_lock", "native"); + LockFactory lockFactory = NoLockFactory.getNoLockFactory(); + if (fsLock.equals("native")) { + // TODO LUCENE MONITOR: this is not needed in next Lucene version + lockFactory = new NativeFSLockFactory(); + } else if (fsLock.equals("simple")) { + lockFactory = new SimpleFSLockFactory(); } + return lockFactory; } - @Override protected void doRenameFile(String from, String to) throws IOException { - File directory = fsDirectory().getDirectory(); + @Override public void renameFile(Directory dir, String from, String to) throws IOException { + File directory = ((FSDirectory) dir).getDirectory(); File old = new File(directory, from); File nu = new File(directory, to); if (nu.exists()) @@ -91,39 +90,13 @@ public FsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStor } } - public abstract FSDirectory fsDirectory(); - - protected LockFactory buildLockFactory() throws IOException { - String fsLock = componentSettings.get("fs_lock", "native"); - LockFactory lockFactory = new NoLockFactory(); - if (fsLock.equals("native")) { - // TODO LUCENE MONITOR: this is not needed in next Lucene version - lockFactory = new NativeFSLockFactory(); - } else if (fsLock.equals("simple")) { - lockFactory = new SimpleFSLockFactory(); - } - return lockFactory; - } - - protected Tuple buildSwitchDirectoryIfNeeded(Directory fsDirectory, ByteBufferCache byteBufferCache) { - boolean cache = componentSettings.getAsBoolean("memory.enabled", false); - if (!cache) { - return null; - } - Directory memDir = new ByteBufferDirectory(byteBufferCache); - // see http://lucene.apache.org/java/3_0_1/fileformats.html - String[] primaryExtensions = componentSettings.getAsArray("memory.extensions", new String[]{"", "del", "gen"}); - if (primaryExtensions == null || primaryExtensions.length == 0) { - return null; - } - Boolean forceUseCompound = null; - for (String extension : primaryExtensions) { - if (!("".equals(extension) || "del".equals(extension) || "gen".equals(extension))) { - // caching internal CFS extension, don't use compound file extension - forceUseCompound = false; - } + @Override public void fullDelete(Directory dir) throws IOException { + FSDirectory fsDirectory = (FSDirectory) dir; + FileSystemUtils.deleteRecursively(fsDirectory.getDirectory()); + // if we are the last ones, delete also the actual index + String[] list = fsDirectory.getDirectory().getParentFile().list(); + if (list == null || list.length == 0) { + FileSystemUtils.deleteRecursively(fsDirectory.getDirectory().getParentFile()); } - - return new Tuple(new SwitchDirectory(ImmutableSet.copyOf(primaryExtensions), memDir, fsDirectory, true), forceUseCompound); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java new file mode 100644 index 0000000000000..336e82d0ca05a --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStore; + +import java.io.File; +import java.io.IOException; + +/** + */ +public class MmapFsDirectoryService extends FsDirectoryService { + + @Inject public MmapFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings, indexStore); + } + + @Override public Directory build() throws IOException { + File location = indexStore.shardIndexLocation(shardId); + FileSystemUtils.mkdirs(location); + return new MMapDirectory(location, buildLockFactory()); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java index 5e8864251b951..ef77976750063 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java @@ -25,7 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.DirectoryService; /** * @author kimchy (shay.banon) @@ -36,7 +36,7 @@ public class MmapFsIndexStore extends FsIndexStore { super(index, indexSettings, indexService, nodeEnv); } - @Override public Class shardStoreClass() { - return MmapFsStore.class; + @Override public Class shardDirectory() { + return MmapFsDirectoryService.class; } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java deleted file mode 100644 index 2b56139ee0bd5..0000000000000 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store.fs; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.MMapDirectory; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.lucene.store.SwitchDirectory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.IndexStore; - -import java.io.File; -import java.io.IOException; - -/** - * @author kimchy (shay.banon) - */ -public class MmapFsStore extends FsStore { - - private final MMapDirectory fsDirectory; - - private final Directory directory; - - private final boolean suggestUseCompoundFile; - - @Inject public MmapFsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException { - super(shardId, indexSettings, indexStore); - LockFactory lockFactory = buildLockFactory(); - File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - this.fsDirectory = new MMapDirectory(location, lockFactory); - - boolean suggestUseCompoundFile; - Tuple switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache); - if (switchDirectory != null) { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - if (switchDirectory.v2() != null) { - suggestUseCompoundFile = switchDirectory.v2(); - } - logger.debug("using [mmap_fs] store with path [{}], cache [true] with extensions [{}]", fsDirectory.getDirectory(), switchDirectory.v1().primaryExtensions()); - directory = wrapDirectory(switchDirectory.v1()); - } else { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - directory = wrapDirectory(fsDirectory); - logger.debug("using [mmap_fs] store with path [{}]", fsDirectory.getDirectory()); - } - this.suggestUseCompoundFile = suggestUseCompoundFile; - } - - @Override public FSDirectory fsDirectory() { - return fsDirectory; - } - - @Override public Directory directory() { - return directory; - } - - @Override public boolean suggestUseCompoundFile() { - return suggestUseCompoundFile; - } -} \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java new file mode 100644 index 0000000000000..5d148347c75fc --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NIOFSDirectory; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStore; + +import java.io.File; +import java.io.IOException; + +/** + */ +public class NioFsDirectoryService extends FsDirectoryService { + + @Inject public NioFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings, indexStore); + } + + @Override public Directory build() throws IOException { + File location = indexStore.shardIndexLocation(shardId); + FileSystemUtils.mkdirs(location); + return new NIOFSDirectory(location, buildLockFactory()); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java index ef5645874ce55..c19ae47854d8b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java @@ -25,7 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.DirectoryService; /** * @author kimchy (shay.banon) @@ -36,7 +36,7 @@ public class NioFsIndexStore extends FsIndexStore { super(index, indexSettings, indexService, nodeEnv); } - @Override public Class shardStoreClass() { - return NioFsStore.class; + @Override public Class shardDirectory() { + return NioFsDirectoryService.class; } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java deleted file mode 100644 index 420caa6286d21..0000000000000 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store.fs; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.NIOFSDirectory; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.lucene.store.SwitchDirectory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.IndexStore; - -import java.io.File; -import java.io.IOException; - -/** - * @author kimchy (shay.banon) - */ -public class NioFsStore extends FsStore { - - private final NIOFSDirectory fsDirectory; - - private final Directory directory; - - private final boolean suggestUseCompoundFile; - - @Inject public NioFsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException { - super(shardId, indexSettings, indexStore); - LockFactory lockFactory = buildLockFactory(); - File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - this.fsDirectory = new NIOFSDirectory(location, lockFactory); - - boolean suggestUseCompoundFile; - Tuple switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache); - if (switchDirectory != null) { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - if (switchDirectory.v2() != null) { - suggestUseCompoundFile = switchDirectory.v2(); - } - logger.debug("using [nio_fs] store with path [{}], cache [true] with extensions [{}]", fsDirectory.getDirectory(), switchDirectory.v1().primaryExtensions()); - directory = wrapDirectory(switchDirectory.v1()); - } else { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - directory = wrapDirectory(fsDirectory); - logger.debug("using [nio_fs] store with path [{}]", fsDirectory.getDirectory()); - } - this.suggestUseCompoundFile = suggestUseCompoundFile; - } - - @Override public FSDirectory fsDirectory() { - return fsDirectory; - } - - @Override public Directory directory() { - return directory; - } - - @Override public boolean suggestUseCompoundFile() { - return suggestUseCompoundFile; - } -} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java new file mode 100644 index 0000000000000..7fec5e31e2e4f --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.store.fs; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.SimpleFSDirectory; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.IndexStore; + +import java.io.File; +import java.io.IOException; + +/** + */ +public class SimpleFsDirectoryService extends FsDirectoryService { + + @Inject public SimpleFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { + super(shardId, indexSettings, indexStore); + } + + @Override public Directory build() throws IOException { + File location = indexStore.shardIndexLocation(shardId); + FileSystemUtils.mkdirs(location); + return new SimpleFSDirectory(location, buildLockFactory()); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java index 2ede53e08d416..bc95287aedd91 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java @@ -25,7 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.DirectoryService; /** * @author kimchy (shay.banon) @@ -36,7 +36,7 @@ public class SimpleFsIndexStore extends FsIndexStore { super(index, indexSettings, indexService, nodeEnv); } - @Override public Class shardStoreClass() { - return SimpleFsStore.class; + @Override public Class shardDirectory() { + return SimpleFsDirectoryService.class; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java deleted file mode 100644 index 9a9511314b893..0000000000000 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store.fs; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.elasticsearch.cache.memory.ByteBufferCache; -import org.elasticsearch.common.collect.Tuple; -import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.common.lucene.store.SwitchDirectory; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.IndexStore; - -import java.io.File; -import java.io.IOException; - -/** - * @author kimchy (shay.banon) - */ -public class SimpleFsStore extends FsStore { - - private SimpleFSDirectory fsDirectory; - - private final Directory directory; - - private final boolean suggestUseCompoundFile; - - @Inject public SimpleFsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException { - super(shardId, indexSettings, indexStore); - LockFactory lockFactory = buildLockFactory(); - File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - this.fsDirectory = new SimpleFSDirectory(location, lockFactory); - - boolean suggestUseCompoundFile; - Tuple switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache); - if (switchDirectory != null) { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - if (switchDirectory.v2() != null) { - suggestUseCompoundFile = switchDirectory.v2(); - } - logger.debug("using [simple_fs] store with path [{}], cache [true] with extensions [{}]", fsDirectory.getDirectory(), switchDirectory.v1().primaryExtensions()); - directory = wrapDirectory(switchDirectory.v1()); - } else { - suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE; - directory = wrapDirectory(fsDirectory); - logger.debug("using [simple_fs] store with path [{}]", fsDirectory.getDirectory()); - } - this.suggestUseCompoundFile = suggestUseCompoundFile; - } - - @Override public FSDirectory fsDirectory() { - return fsDirectory; - } - - @Override public Directory directory() { - return directory; - } - - @Override public boolean suggestUseCompoundFile() { - return suggestUseCompoundFile; - } -} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java similarity index 67% rename from modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferStore.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java index d1a8ef381e0b0..964527a5cc075 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java @@ -27,43 +27,34 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.support.AbstractStore; import java.io.FileNotFoundException; import java.io.IOException; /** - * @author kimchy (shay.banon) */ -public class ByteBufferStore extends AbstractStore { +public class ByteBufferDirectoryService extends AbstractIndexShardComponent implements DirectoryService { - private final CustomByteBufferDirectory bbDirectory; + private final ByteBufferCache byteBufferCache; - private final Directory directory; - - @Inject public ByteBufferStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException { - super(shardId, indexSettings, indexStore); - - this.bbDirectory = new CustomByteBufferDirectory(byteBufferCache); - this.directory = wrapDirectory(bbDirectory); - logger.debug("Using [byte_buffer] store"); + @Inject public ByteBufferDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) { + super(shardId, indexSettings); + this.byteBufferCache = byteBufferCache; } - @Override public Directory directory() { - return directory; + @Override public Directory build() { + return new CustomByteBufferDirectory(byteBufferCache); } - /** - * Its better to not use the compound format when using the Ram store. - */ - @Override public boolean suggestUseCompoundFile() { - return false; + @Override public void renameFile(Directory dir, String from, String to) throws IOException { + ((CustomByteBufferDirectory) dir).renameTo(from, to); } - @Override protected void doRenameFile(String from, String to) throws IOException { - bbDirectory.renameTo(from, to); + @Override public void fullDelete(Directory dir) { } static class CustomByteBufferDirectory extends ByteBufferDirectory { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferIndexStore.java index 505590697f6f7..c1b9aa9c57e3d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferIndexStore.java @@ -27,7 +27,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.support.AbstractIndexStore; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.jvm.JvmStats; @@ -49,8 +49,8 @@ public class ByteBufferIndexStore extends AbstractIndexStore { return false; } - @Override public Class shardStoreClass() { - return ByteBufferStore.class; + @Override public Class shardDirectory() { + return ByteBufferDirectoryService.class; } @Override public ByteSizeValue backingStoreTotalSpace() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java similarity index 65% rename from modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamStore.java rename to modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java index 21d9e48fad90e..5b8dea1d99e2e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java @@ -25,42 +25,30 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; +import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.support.AbstractStore; +import org.elasticsearch.index.store.DirectoryService; import java.io.FileNotFoundException; import java.io.IOException; /** - * @author kimchy (Shay Banon) */ -public class RamStore extends AbstractStore { +public class RamDirectoryService extends AbstractIndexShardComponent implements DirectoryService { - private final CustomRAMDirectory ramDirectory; - - private final Directory directory; - - @Inject public RamStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) throws IOException { - super(shardId, indexSettings, indexStore); - this.ramDirectory = new CustomRAMDirectory(); - this.directory = wrapDirectory(ramDirectory); - logger.debug("Using [ram] Store"); + @Inject public RamDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings) { + super(shardId, indexSettings); } - @Override public Directory directory() { - return directory; + @Override public Directory build() { + return new CustomRAMDirectory(); } - /** - * Its better to not use the compound format when using the Ram store. - */ - @Override public boolean suggestUseCompoundFile() { - return false; + @Override public void renameFile(Directory dir, String from, String to) throws IOException { + ((CustomRAMDirectory) dir).renameTo(from, to); } - @Override protected void doRenameFile(String from, String to) throws IOException { - ramDirectory.renameTo(from, to); + @Override public void fullDelete(Directory dir) { } static class CustomRAMDirectory extends RAMDirectory { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java index 5eb5e80f8c033..abd9b19034ca1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java @@ -25,7 +25,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.store.Store; +import org.elasticsearch.index.store.DirectoryService; import org.elasticsearch.index.store.support.AbstractIndexStore; import org.elasticsearch.monitor.jvm.JvmInfo; import org.elasticsearch.monitor.jvm.JvmStats; @@ -43,8 +43,8 @@ public class RamIndexStore extends AbstractIndexStore { return false; } - @Override public Class shardStoreClass() { - return RamStore.class; + @Override public Class shardDirectory() { + return RamDirectoryService.class; } @Override public ByteSizeValue backingStoreTotalSpace() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java deleted file mode 100644 index 5e355ca8321e1..0000000000000 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java +++ /dev/null @@ -1,517 +0,0 @@ -/* - * Licensed to Elastic Search and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Elastic Search licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.store.support; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Lock; -import org.apache.lucene.store.LockFactory; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; -import org.elasticsearch.common.collect.ImmutableMap; -import org.elasticsearch.common.collect.MapBuilder; -import org.elasticsearch.common.lucene.Directories; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.index.settings.IndexSettings; -import org.elasticsearch.index.shard.AbstractIndexShardComponent; -import org.elasticsearch.index.shard.ShardId; -import org.elasticsearch.index.store.IndexStore; -import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreFileMetaData; -import org.elasticsearch.index.store.StoreStats; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.zip.Adler32; -import java.util.zip.Checksum; - -/** - * @author kimchy (shay.banon) - */ -public abstract class AbstractStore extends AbstractIndexShardComponent implements Store { - - static final String CHECKSUMS_PREFIX = "_checksums-"; - - public static final boolean isChecksum(String name) { - return name.startsWith(CHECKSUMS_PREFIX); - } - - protected final IndexStore indexStore; - - private volatile ImmutableMap filesMetadata = ImmutableMap.of(); - - private volatile String[] files = Strings.EMPTY_ARRAY; - - private final Object mutex = new Object(); - - private final boolean sync; - - protected AbstractStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) { - super(shardId, indexSettings); - this.indexStore = indexStore; - this.sync = componentSettings.getAsBoolean("sync", true); // TODO we don't really need to fsync when using shared gateway... - } - - protected Directory wrapDirectory(Directory dir) throws IOException { - return new StoreDirectory(dir); - } - - @Override public ImmutableMap list() throws IOException { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (String name : files) { - StoreFileMetaData md = metaData(name); - if (md != null) { - builder.put(md.name(), md); - } - } - return builder.build(); - } - - public StoreFileMetaData metaData(String name) throws IOException { - StoreFileMetaData md = filesMetadata.get(name); - if (md == null) { - return null; - } - // IndexOutput not closed, does not exists - if (md.lastModified() == -1 || md.length() == -1) { - return null; - } - return md; - } - - @Override public void deleteContent() throws IOException { - String[] files = directory().listAll(); - IOException lastException = null; - for (String file : files) { - if (isChecksum(file)) { - ((StoreDirectory) directory()).deleteFileChecksum(file); - } else { - try { - directory().deleteFile(file); - } catch (FileNotFoundException e) { - // ignore - } catch (IOException e) { - lastException = e; - } - } - } - if (lastException != null) { - throw lastException; - } - } - - @Override public void fullDelete() throws IOException { - deleteContent(); - } - - @Override public StoreStats stats() throws IOException { - return new StoreStats(Directories.estimateSize(directory())); - } - - @Override public ByteSizeValue estimateSize() throws IOException { - return new ByteSizeValue(Directories.estimateSize(directory())); - } - - @Override public void renameFile(String from, String to) throws IOException { - doRenameFile(from, to); - synchronized (mutex) { - StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one - StoreFileMetaData toMetaData = new StoreFileMetaData(to, fromMetaData.length(), fromMetaData.lastModified(), fromMetaData.checksum()); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(from).put(to, toMetaData).immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - } - - protected abstract void doRenameFile(String from, String to) throws IOException; - - public static Map readChecksums(Directory dir) throws IOException { - long lastFound = -1; - for (String name : dir.listAll()) { - if (!isChecksum(name)) { - continue; - } - long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); - if (current > lastFound) { - lastFound = current; - } - } - if (lastFound == -1) { - return ImmutableMap.of(); - } - IndexInput indexInput = dir.openInput(CHECKSUMS_PREFIX + lastFound); - try { - indexInput.readInt(); // version - return indexInput.readStringStringMap(); - } catch (Exception e) { - // failed to load checksums, ignore and return an empty map - return new HashMap(); - } finally { - indexInput.close(); - } - } - - public void writeChecksums() throws IOException { - writeChecksums((StoreDirectory) directory()); - } - - private void writeChecksums(StoreDirectory dir) throws IOException { - String checksumName = CHECKSUMS_PREFIX + System.currentTimeMillis(); - ImmutableMap files = list(); - synchronized (mutex) { - Map checksums = new HashMap(); - for (StoreFileMetaData metaData : files.values()) { - if (metaData.checksum() != null) { - checksums.put(metaData.name(), metaData.checksum()); - } - } - IndexOutput output = dir.createOutput(checksumName, false); - output.writeInt(0); // version - output.writeStringStringMap(checksums); - output.close(); - } - for (StoreFileMetaData metaData : files.values()) { - if (metaData.name().startsWith(CHECKSUMS_PREFIX) && !checksumName.equals(metaData.name())) { - try { - dir.deleteFileChecksum(metaData.name()); - } catch (Exception e) { - // ignore - } - } - } - } - - /** - * Returns true by default. - */ - @Override public boolean suggestUseCompoundFile() { - return true; - } - - @Override public void close() throws IOException { - directory().close(); - } - - @Override public IndexOutput createOutputWithNoChecksum(String name) throws IOException { - return ((StoreDirectory) directory()).createOutput(name, false); - } - - @Override public void writeChecksum(String name, String checksum) throws IOException { - // update the metadata to include the checksum and write a new checksums file - synchronized (mutex) { - StoreFileMetaData metaData = filesMetadata.get(name); - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), checksum); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); - writeChecksums(); - } - } - - @Override public void writeChecksums(Map checksums) throws IOException { - // update the metadata to include the checksum and write a new checksums file - synchronized (mutex) { - for (Map.Entry entry : checksums.entrySet()) { - StoreFileMetaData metaData = filesMetadata.get(entry.getKey()); - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), entry.getValue()); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(entry.getKey(), metaData).immutableMap(); - } - writeChecksums(); - } - } - - /** - * The idea of the store directory is to cache file level meta data, as well as md5 of it - */ - protected class StoreDirectory extends Directory implements ForceSyncDirectory { - - private final Directory delegate; - - StoreDirectory(Directory delegate) throws IOException { - this.delegate = delegate; - synchronized (mutex) { - Map checksums = readChecksums(delegate); - MapBuilder builder = MapBuilder.newMapBuilder(); - for (String file : delegate.listAll()) { - // BACKWARD CKS SUPPORT - if (file.endsWith(".cks")) { // ignore checksum files here - continue; - } - String checksum = checksums.get(file); - - // BACKWARD CKS SUPPORT - if (checksum == null) { - if (delegate.fileExists(file + ".cks")) { - IndexInput indexInput = delegate.openInput(file + ".cks"); - try { - if (indexInput.length() > 0) { - byte[] checksumBytes = new byte[(int) indexInput.length()]; - indexInput.readBytes(checksumBytes, 0, checksumBytes.length, false); - checksum = Unicode.fromBytes(checksumBytes); - } - } finally { - indexInput.close(); - } - } - } - builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), checksum)); - } - filesMetadata = builder.immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - } - - public Directory delegate() { - return delegate; - } - - @Override public String[] listAll() throws IOException { - return files; - } - - @Override public boolean fileExists(String name) throws IOException { - return filesMetadata.containsKey(name); - } - - @Override public long fileModified(String name) throws IOException { - StoreFileMetaData metaData = filesMetadata.get(name); - if (metaData == null) { - throw new FileNotFoundException(name); - } - // not set yet (IndexOutput not closed) - if (metaData.lastModified() != -1) { - return metaData.lastModified(); - } - return delegate.fileModified(name); - } - - @Override public void touchFile(String name) throws IOException { - delegate.touchFile(name); - synchronized (mutex) { - StoreFileMetaData metaData = filesMetadata.get(name); - if (metaData != null) { - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), delegate.fileModified(name), metaData.checksum()); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); - } - } - } - - public void deleteFileChecksum(String name) throws IOException { - delegate.deleteFile(name); - synchronized (mutex) { - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(name).immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - } - - @Override public void deleteFile(String name) throws IOException { - // we don't allow to delete the checksums files, only using the deleteChecksum method - if (isChecksum(name)) { - return; - } - delegate.deleteFile(name); - synchronized (mutex) { - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(name).immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - } - - @Override public long fileLength(String name) throws IOException { - StoreFileMetaData metaData = filesMetadata.get(name); - if (metaData == null) { - throw new FileNotFoundException(name); - } - // not set yet (IndexOutput not closed) - if (metaData.length() != -1) { - return metaData.length(); - } - return delegate.fileLength(name); - } - - @Override public IndexOutput createOutput(String name) throws IOException { - return createOutput(name, true); - } - - public IndexOutput createOutput(String name, boolean computeChecksum) throws IOException { - IndexOutput out = delegate.createOutput(name); - synchronized (mutex) { - StoreFileMetaData metaData = new StoreFileMetaData(name, -1, -1, null); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - return new StoreIndexOutput(out, name, computeChecksum); - } - - @Override public IndexInput openInput(String name) throws IOException { - return delegate.openInput(name); - } - - @Override public void close() throws IOException { - delegate.close(); - synchronized (mutex) { - filesMetadata = ImmutableMap.of(); - files = Strings.EMPTY_ARRAY; - } - } - - @Override public Lock makeLock(String name) { - return delegate.makeLock(name); - } - - @Override public IndexInput openInput(String name, int bufferSize) throws IOException { - return delegate.openInput(name, bufferSize); - } - - @Override public void clearLock(String name) throws IOException { - delegate.clearLock(name); - } - - @Override public void setLockFactory(LockFactory lockFactory) throws IOException { - delegate.setLockFactory(lockFactory); - } - - @Override public LockFactory getLockFactory() { - return delegate.getLockFactory(); - } - - @Override public String getLockID() { - return delegate.getLockID(); - } - - @Override public void sync(Collection names) throws IOException { - if (sync) { - delegate.sync(names); - } - for (String name : names) { - // write the checksums file when we sync on the segments file (committed) - if (!name.equals("segments.gen") && name.startsWith("segments")) { - writeChecksums(); - break; - } - } - } - - @Override public void sync(String name) throws IOException { - if (sync) { - delegate.sync(name); - } - // write the checksums file when we sync on the segments file (committed) - if (!name.equals("segments.gen") && name.startsWith("segments")) { - writeChecksums(); - } - } - - @Override public void forceSync(String name) throws IOException { - delegate.sync(name); - } - } - - class StoreIndexOutput extends IndexOutput { - - private final IndexOutput delegate; - - private final String name; - - private final Checksum digest; - - StoreIndexOutput(IndexOutput delegate, String name, boolean computeChecksum) { - this.delegate = delegate; - this.name = name; - if (computeChecksum) { - if ("segments.gen".equals(name)) { - // no need to create checksum for segments.gen since its not snapshot to recovery - this.digest = null; - } else if (name.startsWith("segments")) { - // don't compute checksum for segments files, so pure Lucene can open this directory - // and since we, in any case, always recover the segments files - this.digest = null; - } else { -// this.digest = new CRC32(); - // adler is faster, and we compare on length as well, should be enough to check for difference - // between files - this.digest = new Adler32(); - } - } else { - this.digest = null; - } - } - - @Override public void close() throws IOException { - delegate.close(); - String checksum = null; - if (digest != null) { - checksum = Long.toString(digest.getValue(), Character.MAX_RADIX); - } - synchronized (mutex) { - StoreFileMetaData md = new StoreFileMetaData(name, directory().fileLength(name), directory().fileModified(name), checksum); - filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, md).immutableMap(); - files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); - } - } - - @Override public void writeByte(byte b) throws IOException { - delegate.writeByte(b); - if (digest != null) { - digest.update(b); - } - } - - @Override public void writeBytes(byte[] b, int offset, int length) throws IOException { - delegate.writeBytes(b, offset, length); - if (digest != null) { - digest.update(b, offset, length); - } - } - - // don't override it, base class method simple reads from input and writes to this output -// @Override public void copyBytes(IndexInput input, long numBytes) throws IOException { -// delegate.copyBytes(input, numBytes); -// } - - @Override public void flush() throws IOException { - delegate.flush(); - } - - @Override public long getFilePointer() { - return delegate.getFilePointer(); - } - - @Override public void seek(long pos) throws IOException { - // seek might be called on files, which means that the checksum is not file checksum - // but a checksum of the bytes written to this stream, which is the same for each - // type of file in lucene - delegate.seek(pos); - } - - @Override public long length() throws IOException { - return delegate.length(); - } - - @Override public void setLength(long length) throws IOException { - delegate.setLength(length); - } - - @Override public void writeStringStringMap(Map map) throws IOException { - delegate.writeStringStringMap(map); - } - } -} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index b26b24c82b76e..8bbff0d078a98 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -42,7 +42,7 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.shard.service.InternalIndexShard; -import org.elasticsearch.index.store.support.AbstractStore; +import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndexMissingException; import org.elasticsearch.indices.IndicesLifecycle; @@ -452,7 +452,7 @@ class CleanFilesRequestHandler extends BaseTransportRequestHandler checksums = null; try { - checksums = AbstractStore.readChecksums(directory); + checksums = Store.readChecksums(directory); for (File file : indexFile.listFiles()) { // BACKWARD CKS SUPPORT if (file.getName().endsWith(".cks")) { continue; } - if (AbstractStore.isChecksum(file.getName())) { + if (Store.isChecksum(file.getName())) { continue; } files.put(file.getName(), new StoreFileMetaData(file.getName(), file.length(), file.lastModified(), checksums.get(file.getName()))); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/engine/AbstractSimpleEngineTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/engine/AbstractSimpleEngineTests.java index 9c4db2add2f15..9f1046be1b88d 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/engine/AbstractSimpleEngineTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/engine/AbstractSimpleEngineTests.java @@ -39,7 +39,7 @@ import org.elasticsearch.index.settings.IndexSettingsService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.ram.RamStore; +import org.elasticsearch.index.store.ram.RamDirectoryService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.fs.FsTranslog; import org.testng.annotations.AfterMethod; @@ -96,11 +96,11 @@ public abstract class AbstractSimpleEngineTests { } protected Store createStore() throws IOException { - return new RamStore(shardId, EMPTY_SETTINGS, null); + return new Store(shardId, EMPTY_SETTINGS, null, new RamDirectoryService(shardId, EMPTY_SETTINGS)); } protected Store createStoreReplica() throws IOException { - return new RamStore(shardId, EMPTY_SETTINGS, null); + return new Store(shardId, EMPTY_SETTINGS, null, new RamDirectoryService(shardId, EMPTY_SETTINGS)); } protected Translog createTranslog() { From 5ba6ec5a6988e0eac1a7e20f631b3023a1d8efcb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 17 Sep 2011 00:49:21 +0300 Subject: [PATCH 29/96] Versioning: Delete on an already deleted document should still affect versioning, closes #1341. --- .../index/engine/robin/RobinEngine.java | 12 +++++--- .../versioning/SimpleVersioningTests.java | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index cda00abf59451..8e1b455016221 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -654,11 +654,15 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { } if (currentVersion == -1) { - // if the doc does not exists, just update with doc 0 - delete.version(0).notFound(true); + // doc does not exists and no prior deletes + delete.version(updatedVersion).notFound(true); + Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); + versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); } else if (versionValue != null && versionValue.delete()) { - // if its a delete on delete and we have the current delete version, return it - delete.version(versionValue.version()).notFound(true); + // a "delete on delete", in this case, we still increment the version, log it, and return that version + delete.version(updatedVersion).notFound(true); + Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); + versionMap.put(delete.uid().text(), new VersionValue(updatedVersion, true, threadPool.estimatedTimeInMillis(), translogLocation)); } else { delete.version(updatedVersion); writer.deleteDocuments(delete.uid()); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java index 0bd3df0819b64..de9f1ab00f9d5 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteResponse; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -85,6 +86,20 @@ public class SimpleVersioningTests extends AbstractNodesTests { for (int i = 0; i < 10; i++) { assertThat(client.prepareGet("test", "type", "1").execute().actionGet().version(), equalTo(14l)); } + + DeleteResponse deleteResponse = client2.prepareDelete("test", "type", "1").setVersion(17).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + assertThat(deleteResponse.notFound(), equalTo(false)); + assertThat(deleteResponse.version(), equalTo(17l)); + + try { + client2.prepareDelete("test", "type", "1").setVersion(2).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + } catch (ElasticSearchException e) { + assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class)); + } + + deleteResponse = client2.prepareDelete("test", "type", "1").setVersion(18).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + assertThat(deleteResponse.notFound(), equalTo(true)); + assertThat(deleteResponse.version(), equalTo(18l)); } @Test public void testSimpleVersioning() throws Exception { @@ -154,6 +169,20 @@ public class SimpleVersioningTests extends AbstractNodesTests { SearchResponse searchResponse = client.prepareSearch().setQuery(matchAllQuery()).execute().actionGet(); assertThat(searchResponse.hits().getAt(0).version(), equalTo(-1l)); } + + DeleteResponse deleteResponse = client2.prepareDelete("test", "type", "1").setVersion(2).execute().actionGet(); + assertThat(deleteResponse.notFound(), equalTo(false)); + assertThat(deleteResponse.version(), equalTo(3l)); + + try { + client2.prepareDelete("test", "type", "1").setVersion(2).execute().actionGet(); + } catch (ElasticSearchException e) { + assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class)); + } + + deleteResponse = client2.prepareDelete("test", "type", "1").setVersion(3).execute().actionGet(); + assertThat(deleteResponse.notFound(), equalTo(true)); + assertThat(deleteResponse.version(), equalTo(4l)); } @Test public void testSimpleVersioningWithFlush() throws Exception { From 0977b793da6f9f9bf98c4981d2bd571be839ac21 Mon Sep 17 00:00:00 2001 From: Austin McKinley Date: Fri, 16 Sep 2011 14:15:55 -0700 Subject: [PATCH 30/96] adding timeout to list of connection failures --- .../elasticsearch/common/transport/NetworkExceptionHelper.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java index 71f323b596f04..283df64487a14 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/transport/NetworkExceptionHelper.java @@ -52,6 +52,9 @@ public static boolean isCloseConnectionException(Throwable e) { if (e.getMessage().contains("Broken pipe")) { return true; } + if (e.getMessage().contains("Connection timed out")) { + return true; + } } return false; } From 305cf4a5671cf4390ed0e9d5db15b5e068370aa8 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 17 Sep 2011 02:21:20 +0300 Subject: [PATCH 31/96] Bulk API: Properly retry execution on temporal state changes, closes #1343. --- .../action/bulk/TransportShardBulkAction.java | 9 +++ ...nsportShardReplicationOperationAction.java | 67 ++++++++++--------- 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 116a288f8d7a7..7c76335b01f31 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.bulk; +import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.RoutingMissingException; @@ -158,6 +159,10 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation responses[i] = new BulkItemResponse(item.id(), indexRequest.opType().toString().toLowerCase(), new IndexResponse(indexRequest.index(), indexRequest.type(), indexRequest.id(), version)); } catch (Exception e) { + // rethrow the failure if we are going to retry on primary and let parent failure to handle it + if (retryPrimaryException(e)) { + throw (ElasticSearchException) e; + } logger.debug("[{}][{}] failed to bulk item (index) {}", e, shardRequest.request.index(), shardRequest.shardId, indexRequest); responses[i] = new BulkItemResponse(item.id(), indexRequest.opType().toString().toLowerCase(), new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), ExceptionsHelper.detailedMessage(e))); @@ -174,6 +179,10 @@ public class TransportShardBulkAction extends TransportShardReplicationOperation responses[i] = new BulkItemResponse(item.id(), "delete", new DeleteResponse(deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), delete.version(), delete.notFound())); } catch (Exception e) { + // rethrow the failure if we are going to retry on primary and let parent failure to handle it + if (retryPrimaryException(e)) { + throw (ElasticSearchException) e; + } logger.debug("[{}][{}] failed to bulk item (delete) {}", e, shardRequest.request.index(), shardRequest.shardId, deleteRequest); responses[i] = new BulkItemResponse(item.id(), "delete", new BulkItemResponse.Failure(deleteRequest.index(), deleteRequest.type(), deleteRequest.id(), ExceptionsHelper.detailedMessage(e))); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java index 5a5935d4ce83d..3a5491eb959eb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/support/replication/TransportShardReplicationOperationAction.java @@ -160,6 +160,42 @@ protected IndexShard indexShard(ShardOperationRequest shardRequest) { return indicesService.indexServiceSafe(shardRequest.request.index()).shardSafe(shardRequest.shardId); } + protected boolean retryPrimaryException(Throwable e) { + Throwable cause = ExceptionsHelper.unwrapCause(e); + return cause instanceof IndexShardMissingException || + cause instanceof IllegalIndexShardStateException || + cause instanceof IndexMissingException; + } + + /** + * Should an exception be ignored when the operation is performed on the replica. + */ + boolean ignoreReplicaException(Throwable e) { + Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof IllegalIndexShardStateException) { + return true; + } + if (cause instanceof IndexMissingException) { + return true; + } + if (cause instanceof IndexShardMissingException) { + return true; + } + if (cause instanceof ConnectTransportException) { + return true; + } + // on version conflict or document missing, it means + // that a news change has crept into the replica, and its fine + if (cause instanceof VersionConflictEngineException) { + return true; + } + // same here + if (cause instanceof DocumentAlreadyExistsEngineException) { + return true; + } + return false; + } + class OperationTransportHandler extends BaseTransportRequestHandler { @Override public Request newInstance() { @@ -429,7 +465,7 @@ void performOnPrimary(int primaryShardId, boolean fromDiscoveryListener, final S performReplicas(response); } catch (Exception e) { // shard has not been allocated yet, retry it here - if (e instanceof IndexShardMissingException || e instanceof IllegalIndexShardStateException || e instanceof IndexMissingException) { + if (retryPrimaryException(e)) { retry(fromDiscoveryListener, shard.shardId()); return; } @@ -572,35 +608,6 @@ private void finishIfPossible() { } } } - - /** - * Should an exception be ignored when the operation is performed on the replica. - */ - boolean ignoreReplicaException(Throwable e) { - Throwable cause = ExceptionsHelper.unwrapCause(e); - if (cause instanceof IllegalIndexShardStateException) { - return true; - } - if (cause instanceof IndexMissingException) { - return true; - } - if (cause instanceof IndexShardMissingException) { - return true; - } - if (cause instanceof ConnectTransportException) { - return true; - } - // on version conflict or document missing, it means - // that a news change has crept into the replica, and its fine - if (cause instanceof VersionConflictEngineException) { - return true; - } - // same here - if (cause instanceof DocumentAlreadyExistsEngineException) { - return true; - } - return false; - } } public static class PrimaryResponse { From b66a3b7c5974d2d1b9a843817713528aac5ddb59 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Sep 2011 13:44:08 +0300 Subject: [PATCH 32/96] Realtime Get: Under high concurrent indexing and immediate get, a get might be missed while flushing, closes #1344. --- .../index/engine/robin/RobinEngine.java | 16 +++- .../test/stress/get/GetStressTest.java | 96 +++++++++++++++++++ .../stress/{mget => get}/MGetStress1.java | 2 +- 3 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/GetStressTest.java rename modules/test/integration/src/test/java/org/elasticsearch/test/stress/{mget => get}/MGetStress1.java (98%) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 8e1b455016221..5fb3356b5ced5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -783,7 +783,7 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { } try { - + boolean makeTransientCurrent = false; if (flush.full()) { rwl.writeLock().lock(); try { @@ -845,10 +845,10 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { // we did not commit anything, revert to the old translog translog.revertTransient(); } else { - translog.makeTransientCurrent(); + makeTransientCurrent = true; } } else { - translog.makeTransientCurrent(); + makeTransientCurrent = true; } } catch (Exception e) { translog.revertTransient(); @@ -864,12 +864,20 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { } } refreshVersioningTable(threadPool.estimatedTimeInMillis()); + // we need to move transient to current only after we refresh + // so items added to current will still be around for realtime get + // when tans overrides it + if (makeTransientCurrent) { + translog.makeTransientCurrent(); + } try { SegmentInfos infos = new SegmentInfos(); infos.read(store.directory()); lastCommittedSegmentInfos = infos; } catch (Exception e) { - logger.warn("failed to read latest segment infos on flush", e); + if (!closed) { + logger.warn("failed to read latest segment infos on flush", e); + } } } finally { flushing.set(false); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/GetStressTest.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/GetStressTest.java new file mode 100644 index 0000000000000..4220a57f20fff --- /dev/null +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/GetStressTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.stress.get; + +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class GetStressTest { + + public static void main(String[] args) throws Exception { + Settings settings = ImmutableSettings.settingsBuilder() + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 1) + .build(); + + final int NUMBER_OF_NODES = 2; + final int NUMBER_OF_THREADS = 50; + final TimeValue TEST_TIME = TimeValue.parseTimeValue("10m", null); + + Node[] nodes = new Node[NUMBER_OF_NODES]; + for (int i = 0; i < nodes.length; i++) { + nodes[i] = NodeBuilder.nodeBuilder().settings(settings).node(); + } + + final Node client = NodeBuilder.nodeBuilder() + .settings(settings) + .client(true) + .node(); + + client.client().admin().indices().prepareCreate("test").execute().actionGet(); + + final AtomicBoolean done = new AtomicBoolean(); + final AtomicLong idGenerator = new AtomicLong(); + final AtomicLong counter = new AtomicLong(); + + Thread[] threads = new Thread[NUMBER_OF_THREADS]; + for (int i = 0; i < threads.length; i++) { + threads[i] = new Thread(new Runnable() { + @Override public void run() { + ThreadLocalRandom random = ThreadLocalRandom.current(); + while (!done.get()) { + String id = String.valueOf(idGenerator.incrementAndGet()); + client.client().prepareIndex("test", "type1", id) + .setSource("field", random.nextInt(100)) + .execute().actionGet(); + + GetResponse getResponse = client.client().prepareGet("test", "type1", id) + //.setFields(Strings.EMPTY_ARRAY) + .execute().actionGet(); + if (!getResponse.exists()) { + System.err.println("Failed to find " + id); + } + + long count = counter.incrementAndGet(); + if ((count % 10000) == 0) { + System.out.println("Executed " + count); + } + } + } + }); + } + for (Thread thread : threads) { + thread.start(); + } + + Thread.sleep(TEST_TIME.millis()); + + System.out.println("test done."); + done.set(true); + } +} \ No newline at end of file diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/mget/MGetStress1.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/MGetStress1.java similarity index 98% rename from modules/test/integration/src/test/java/org/elasticsearch/test/stress/mget/MGetStress1.java rename to modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/MGetStress1.java index fa3f18dbea1f8..f7262f66d9ebf 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/mget/MGetStress1.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/get/MGetStress1.java @@ -17,7 +17,7 @@ * under the License. */ -package org.elasticsearch.test.stress.mget; +package org.elasticsearch.test.stress.get; import org.elasticsearch.action.get.MultiGetItemResponse; import org.elasticsearch.action.get.MultiGetResponse; From 52ca63deb9fc7a96f18fb2db19bd648dbee87cc1 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Sep 2011 16:11:17 +0300 Subject: [PATCH 33/96] move refreshing version table to be done right after the commit within the respective lock, for full flush (rarely used) its needed... --- .../index/engine/robin/RobinEngine.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 5fb3356b5ced5..241ec97db9de2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -248,6 +248,8 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { } try { + // commit on a just opened writer will commit even if there are no changes done to it + // we rely on that for the commit data translog id key if (IndexReader.indexExists(store.directory())) { Map commitUserData = IndexReader.getCommitUserData(store.directory()); if (commitUserData.containsKey(Translog.TRANSLOG_ID_KEY)) { @@ -802,6 +804,8 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { indexWriter.close(false); indexWriter = createWriter(); + // commit on a just opened writer will commit even if there are no changes done to it + // we rely on that for the commit data translog id key if (flushNeeded || flush.force()) { flushNeeded = false; long translogId = translogIdGenerator.incrementAndGet(); @@ -812,6 +816,8 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { AcquirableResource current = nrtResource; nrtResource = buildNrtResource(indexWriter); current.markForClose(); + + refreshVersioningTable(threadPool.estimatedTimeInMillis()); } catch (Exception e) { throw new FlushFailedEngineException(shardId, e); } catch (OutOfMemoryError e) { @@ -850,6 +856,13 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { } else { makeTransientCurrent = true; } + if (makeTransientCurrent) { + refreshVersioningTable(threadPool.estimatedTimeInMillis()); + // we need to move transient to current only after we refresh + // so items added to current will still be around for realtime get + // when tans overrides it + translog.makeTransientCurrent(); + } } catch (Exception e) { translog.revertTransient(); throw new FlushFailedEngineException(shardId, e); @@ -863,13 +876,6 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { rwl.readLock().unlock(); } } - refreshVersioningTable(threadPool.estimatedTimeInMillis()); - // we need to move transient to current only after we refresh - // so items added to current will still be around for realtime get - // when tans overrides it - if (makeTransientCurrent) { - translog.makeTransientCurrent(); - } try { SegmentInfos infos = new SegmentInfos(); infos.read(store.directory()); From f072c7e91b8e36c6d6e73866c508db6ab938afd3 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Sep 2011 21:35:01 +0300 Subject: [PATCH 34/96] add meter metric implementation --- .../elasticsearch/common/metrics/EWMA.java | 119 +++++++++++++++++ .../common/metrics/MeterMetric.java | 123 ++++++++++++++++++ 2 files changed, 242 insertions(+) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/EWMA.java create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/EWMA.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/EWMA.java new file mode 100644 index 0000000000000..d16275fcb8546 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/EWMA.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.metrics; + +import org.elasticsearch.common.util.concurrent.jsr166e.LongAdder; + +import java.util.concurrent.TimeUnit; + +/** + * An exponentially-weighted moving average. + * + * @see UNIX Load Average Part 1: How It Works + * @see UNIX Load Average Part 2: Not Your Average Average + * + * Taken from codahale metric module, changed to use LongAdder + */ +public class EWMA { + private static final double M1_ALPHA = 1 - Math.exp(-5 / 60.0); + private static final double M5_ALPHA = 1 - Math.exp(-5 / 60.0 / 5); + private static final double M15_ALPHA = 1 - Math.exp(-5 / 60.0 / 15); + + private volatile boolean initialized = false; + private volatile double rate = 0.0; + + private final LongAdder uncounted = new LongAdder(); + private final double alpha, interval; + + /** + * Creates a new EWMA which is equivalent to the UNIX one minute load average and which expects to be ticked every + * 5 seconds. + * + * @return a one-minute EWMA + */ + public static EWMA oneMinuteEWMA() { + return new EWMA(M1_ALPHA, 5, TimeUnit.SECONDS); + } + + /** + * Creates a new EWMA which is equivalent to the UNIX five minute load average and which expects to be ticked every + * 5 seconds. + * + * @return a five-minute EWMA + */ + public static EWMA fiveMinuteEWMA() { + return new EWMA(M5_ALPHA, 5, TimeUnit.SECONDS); + } + + /** + * Creates a new EWMA which is equivalent to the UNIX fifteen minute load average and which expects to be ticked + * every 5 seconds. + * + * @return a fifteen-minute EWMA + */ + public static EWMA fifteenMinuteEWMA() { + return new EWMA(M15_ALPHA, 5, TimeUnit.SECONDS); + } + + /** + * Create a new EWMA with a specific smoothing constant. + * + * @param alpha the smoothing constant + * @param interval the expected tick interval + * @param intervalUnit the time unit of the tick interval + */ + public EWMA(double alpha, long interval, TimeUnit intervalUnit) { + this.interval = intervalUnit.toNanos(interval); + this.alpha = alpha; + } + + /** + * Update the moving average with a new value. + * + * @param n the new value + */ + public void update(long n) { + uncounted.add(n); + } + + /** + * Mark the passage of time and decay the current rate accordingly. + */ + public void tick() { + final long count = uncounted.sumThenReset(); + double instantRate = count / interval; + if (initialized) { + rate += (alpha * (instantRate - rate)); + } else { + rate = instantRate; + initialized = true; + } + } + + /** + * Returns the rate in the given units of time. + * + * @param rateUnit the unit of time + * @return the rate + */ + public double rate(TimeUnit rateUnit) { + return rate * (double) rateUnit.toNanos(1); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java new file mode 100644 index 0000000000000..52b47207704dc --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/metrics/MeterMetric.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.metrics; + +import org.elasticsearch.common.util.concurrent.jsr166e.LongAdder; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * A meter metric which measures mean throughput and one-, five-, and + * fifteen-minute exponentially-weighted moving average throughputs. + * + * @see EMA + * + * taken from codahale metric module, replaced with LongAdder + */ +public class MeterMetric implements Metric { + private static final long INTERVAL = 5; // seconds + + private final EWMA m1Rate = EWMA.oneMinuteEWMA(); + private final EWMA m5Rate = EWMA.fiveMinuteEWMA(); + private final EWMA m15Rate = EWMA.fifteenMinuteEWMA(); + + private final LongAdder count = new LongAdder(); + private final long startTime = System.nanoTime(); + private final TimeUnit rateUnit; + private final ScheduledFuture future; + + public MeterMetric(ScheduledExecutorService tickThread, TimeUnit rateUnit) { + this.rateUnit = rateUnit; + this.future = tickThread.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + tick(); + } + }, INTERVAL, INTERVAL, TimeUnit.SECONDS); + } + + public TimeUnit rateUnit() { + return rateUnit; + } + + /** + * Updates the moving averages. + */ + void tick() { + m1Rate.tick(); + m5Rate.tick(); + m15Rate.tick(); + } + + /** + * Mark the occurrence of an event. + */ + public void mark() { + mark(1); + } + + /** + * Mark the occurrence of a given number of events. + * + * @param n the number of events + */ + public void mark(long n) { + count.add(n); + m1Rate.update(n); + m5Rate.update(n); + m15Rate.update(n); + } + + public long count() { + return count.sum(); + } + + public double fifteenMinuteRate() { + return m15Rate.rate(rateUnit); + } + + public double fiveMinuteRate() { + return m5Rate.rate(rateUnit); + } + + public double meanRate() { + long count = count(); + if (count == 0) { + return 0.0; + } else { + final long elapsed = (System.nanoTime() - startTime); + return convertNsRate(count / (double) elapsed); + } + } + + public double oneMinuteRate() { + return m1Rate.rate(rateUnit); + } + + private double convertNsRate(double ratePerNs) { + return ratePerNs * (double) rateUnit.toNanos(1); + } + + public void stop() { + future.cancel(false); + } +} From dc85e227af6ac0c8c60773ac98dd5704fc912952 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Sep 2011 21:50:16 +0300 Subject: [PATCH 35/96] upgrade to latest jsr166 code --- modules/jarjar/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/jarjar/build.gradle b/modules/jarjar/build.gradle index 3976f170d2b4e..c6e09be290de9 100644 --- a/modules/jarjar/build.gradle +++ b/modules/jarjar/build.gradle @@ -6,8 +6,8 @@ dependencies { runtime 'com.googlecode.concurrentlinkedhashmap:concurrentlinkedhashmap-lru:1.2' runtime 'com.google.guava:guava:r09' runtime 'org.elasticsearch:es-trove:3.0.0' - runtime 'org.elasticsearch:es-jsr166y:20110820' - runtime 'org.elasticsearch:es-jsr166e:20110820' + runtime 'org.elasticsearch:es-jsr166y:20110918' + runtime 'org.elasticsearch:es-jsr166e:20110918' runtime 'commons-codec:commons-codec:1.5' From f3106de23ff638ae5d62bdcb4495dc0e2fe77ff7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 18 Sep 2011 21:51:42 +0300 Subject: [PATCH 36/96] suppress unchecked compile warnings --- .../script/groovy/GroovyScriptEngineService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/lang/groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java b/plugins/lang/groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java index f34af7acc58de..5fedebc761fea 100644 --- a/plugins/lang/groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java +++ b/plugins/lang/groovy/src/main/java/org/elasticsearch/script/groovy/GroovyScriptEngineService.java @@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicLong; /** - * @author kimchy (shay.banon) */ public class GroovyScriptEngineService extends AbstractComponent implements ScriptEngineService { @@ -83,6 +82,7 @@ public class GroovyScriptEngineService extends AbstractComponent implements Scri } } + @SuppressWarnings({"unchecked"}) @Override public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { try { Class scriptClass = (Class) compiledScript; @@ -127,6 +127,7 @@ public GroovyExecutableScript(Script script) { this.script = script; } + @SuppressWarnings({"unchecked"}) @Override public void setNextVar(String name, Object value) { script.getBinding().getVariables().put(name, value); } @@ -163,10 +164,12 @@ public GroovySearchScript(Script script, SearchLookup lookup) { lookup.setNextDocId(doc); } + @SuppressWarnings({"unchecked"}) @Override public void setNextScore(float score) { script.getBinding().getVariables().put("_score", score); } + @SuppressWarnings({"unchecked"}) @Override public void setNextVar(String name, Object value) { script.getBinding().getVariables().put(name, value); } From 29d0bfdaa9d8454f6a7757ed6c4b0b1fcb9f05f5 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 19 Sep 2011 17:45:44 +0300 Subject: [PATCH 37/96] don't log delete of unused shard location if it does not exists --- .../java/org/elasticsearch/indices/store/IndicesStore.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index 3df45549f6490..f140a2abed5a0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -131,8 +131,11 @@ public void close() { } if (shardCanBeDeleted) { ShardId shardId = indexShardRoutingTable.shardId(); - logger.debug("[{}][{}] deleting shard that is no longer used", shardId.index().name(), shardId.id()); - FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(shardId)); + File shardLocation = nodeEnv.shardLocation(shardId); + if (shardLocation.exists()) { + logger.debug("[{}][{}] deleting shard that is no longer used", shardId.index().name(), shardId.id()); + FileSystemUtils.deleteRecursively(shardLocation); + } } } } From 9a13763315e8da781bf7f7b6e12c8819f9271513 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 20 Sep 2011 14:33:02 +0300 Subject: [PATCH 38/96] Improve source based fields loading when searching, closes #1347. --- .../xcontent/support/XContentMapValues.java | 85 ++++++++++++++++ .../search/fetch/FetchPhase.java | 85 +++++++++------- .../search/fetch/FieldsParseElement.java | 34 +------ .../search/highlight/HighlightPhase.java | 2 +- .../SourceScoreOrderFragmentsBuilder.java | 2 +- .../SourceSimpleFragmentsBuilder.java | 2 +- .../search/lookup/SourceLookup.java | 44 +-------- .../support/XContentMapValuesTests.java | 99 +++++++++++++++++++ .../search/fields/SearchFieldsTests.java | 2 - 9 files changed, 247 insertions(+), 108 deletions(-) create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java index e565104972bb6..26c80cd0a9370 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java @@ -19,8 +19,11 @@ package org.elasticsearch.common.xcontent.support; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.unit.TimeValue; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -29,6 +32,88 @@ */ public class XContentMapValues { + /** + * Extracts raw values (string, int, and so on) based on the path provided returning all of them + * as a single list. + */ + public static List extractRawValues(String path, Map map) { + List values = Lists.newArrayList(); + String[] pathElements = Strings.splitStringToArray(path, '.'); + if (pathElements.length == 0) { + return values; + } + extractRawValues(values, map, pathElements, 0); + return values; + } + + @SuppressWarnings({"unchecked"}) + private static void extractRawValues(List values, Map part, String[] pathElements, int index) { + if (index == pathElements.length) { + return; + } + String currentPath = pathElements[index]; + Object currentValue = part.get(currentPath); + if (currentValue == null) { + return; + } + if (currentValue instanceof Map) { + extractRawValues(values, (Map) currentValue, pathElements, index + 1); + } else if (currentValue instanceof List) { + extractRawValues(values, (List) currentValue, pathElements, index + 1); + } else { + values.add(currentValue); + } + } + + @SuppressWarnings({"unchecked"}) + private static void extractRawValues(List values, List part, String[] pathElements, int index) { + for (Object value : part) { + if (value == null) { + continue; + } + if (value instanceof Map) { + extractRawValues(values, (Map) value, pathElements, index); + } else if (value instanceof List) { + extractRawValues(values, (List) value, pathElements, index); + } else { + values.add(value); + } + } + } + + public static Object extractValue(String path, Map map) { + String[] pathElements = Strings.splitStringToArray(path, '.'); + if (pathElements.length == 0) { + return null; + } + return extractValue(pathElements, 0, map); + } + + @SuppressWarnings({"unchecked"}) private static Object extractValue(String[] pathElements, int index, Object currentValue) { + if (index == pathElements.length) { + return currentValue; + } + if (currentValue == null) { + return null; + } + if (currentValue instanceof Map) { + Map map = (Map) currentValue; + return extractValue(pathElements, index + 1, map.get(pathElements[index])); + } + if (currentValue instanceof List) { + List valueList = (List) currentValue; + List newList = new ArrayList(valueList.size()); + for (Object o : valueList) { + Object listValue = extractValue(pathElements, index, o); + if (listValue != null) { + newList.add(listValue); + } + } + return newList; + } + return null; + } + public static boolean isObject(Object node) { return node instanceof Map; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index ecd22b727e326..3f54f1c04e55b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -23,6 +23,7 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.document.ResetFieldSelector; import org.elasticsearch.index.Index; @@ -81,7 +82,35 @@ public class FetchPhase implements SearchPhase { } public void execute(SearchContext context) { - ResetFieldSelector fieldSelector = buildFieldSelectors(context); + ResetFieldSelector fieldSelector; + List extractFieldNames = null; + if (context.hasScriptFields() && !context.hasFieldNames()) { + // we ask for script fields, and no field names, don't load the source + fieldSelector = UidFieldSelector.INSTANCE; + } else if (!context.hasFieldNames()) { + fieldSelector = new UidAndSourceFieldSelector(); + } else if (context.fieldNames().isEmpty()) { + fieldSelector = UidFieldSelector.INSTANCE; + } else if (context.fieldNames().get(0).equals("*")) { + // asked for all stored fields, just return null so all of them will be loaded + // don't load the source field in this case, makes little sense to get it with all stored fields + fieldSelector = AllButSourceFieldSelector.INSTANCE; + } else { + FieldMappersFieldSelector fieldSelectorMapper = new FieldMappersFieldSelector(); + for (String fieldName : context.fieldNames()) { + FieldMappers x = context.mapperService().smartNameFieldMappers(fieldName); + if (x != null && x.mapper().stored()) { + fieldSelectorMapper.add(x); + } else { + if (extractFieldNames == null) { + extractFieldNames = Lists.newArrayList(); + } + extractFieldNames.add(fieldName); + } + } + fieldSelectorMapper.add(UidFieldMapper.NAME); + fieldSelector = fieldSelectorMapper; + } InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()]; for (int index = 0; index < context.docIdsToLoadSize(); index++) { @@ -148,6 +177,28 @@ public void execute(SearchContext context) { int readerIndex = context.searcher().readerIndex(docId); IndexReader subReader = context.searcher().subReaders()[readerIndex]; int subDoc = docId - context.searcher().docStarts()[readerIndex]; + + // go over and extract fields that are not mapped / stored + context.lookup().setNextReader(subReader); + context.lookup().setNextDocId(docId); + if (extractFieldNames != null) { + for (String extractFieldName : extractFieldNames) { + Object value = context.lookup().source().extractValue(extractFieldName); + if (value != null) { + if (searchHit.fieldsOrNull() == null) { + searchHit.fields(new HashMap(2)); + } + + SearchHitField hitField = searchHit.fields().get(extractFieldName); + if (hitField == null) { + hitField = new InternalSearchHitField(extractFieldName, new ArrayList(2)); + searchHit.fields().put(extractFieldName, hitField); + } + hitField.values().add(value); + } + } + } + for (SearchHitPhase hitPhase : hitPhases) { SearchHitPhase.HitContext hitContext = new SearchHitPhase.HitContext(); if (hitPhase.executionNeeded(context)) { @@ -189,36 +240,4 @@ private Document loadDocument(SearchContext context, ResetFieldSelector fieldSel throw new FetchPhaseExecutionException(context, "Failed to fetch doc id [" + docId + "]", e); } } - - private ResetFieldSelector buildFieldSelectors(SearchContext context) { - if (context.hasScriptFields() && !context.hasFieldNames()) { - // we ask for script fields, and no field names, don't load the source - return UidFieldSelector.INSTANCE; - } - - if (!context.hasFieldNames()) { - return new UidAndSourceFieldSelector(); - } - - if (context.fieldNames().isEmpty()) { - return UidFieldSelector.INSTANCE; - } - - // asked for all stored fields, just return null so all of them will be loaded - // don't load the source field in this case, makes little sense to get it with all stored fields - if (context.fieldNames().get(0).equals("*")) { - return AllButSourceFieldSelector.INSTANCE; - } - - FieldMappersFieldSelector fieldSelector = new FieldMappersFieldSelector(); - for (String fieldName : context.fieldNames()) { - FieldMappers x = context.mapperService().smartNameFieldMappers(fieldName); - if (x == null) { - throw new FetchPhaseExecutionException(context, "No mapping for field [" + fieldName + "] in order to load it"); - } - fieldSelector.add(x); - } - fieldSelector.add(UidFieldMapper.NAME); - return fieldSelector; - } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java index 2b8edc153706f..3dec32bc98be1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/fetch/FieldsParseElement.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.fetch; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.SearchParseElement; import org.elasticsearch.search.fetch.script.ScriptFieldsContext; @@ -42,21 +41,8 @@ public class FieldsParseElement implements SearchParseElement { SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", name, null); context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true)); } else { - if ("*".equals(name)) { - added = true; - context.fieldNames().add("*"); - } else { - FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(name); - if (fieldMapper != null) { - if (fieldMapper.stored()) { - added = true; - context.fieldNames().add(name); - } else { - SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", "_source." + fieldMapper.names().fullName(), null); - context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true)); - } - } - } + added = true; + context.fieldNames().add(name); } } if (!added) { @@ -69,21 +55,7 @@ public class FieldsParseElement implements SearchParseElement { SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", name, null); context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true)); } else { - if ("*".equals(name)) { - context.fieldNames().add("*"); - } else { - FieldMapper fieldMapper = context.mapperService().smartNameFieldMapper(name); - if (fieldMapper != null) { - if (fieldMapper.stored()) { - context.fieldNames().add(name); - } else { - SearchScript searchScript = context.scriptService().search(context.lookup(), "mvel", "_source." + fieldMapper.names().fullName(), null); - context.scriptFields().add(new ScriptFieldsContext.ScriptField(name, searchScript, true)); - } - } else { - context.emptyFieldNames(); // don't load anything if we can't find mapping - } - } + context.fieldNames().add(name); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java index 0db8088edb3be..5f80520aadf7a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java @@ -141,7 +141,7 @@ public static class Encoders { SearchLookup lookup = context.lookup(); lookup.setNextReader(hitContext.reader()); lookup.setNextDocId(hitContext.docId()); - textsToHighlight = lookup.source().getValues(mapper.names().fullName()); + textsToHighlight = lookup.source().extractRawValues(mapper.names().fullName()); } // a HACK to make highlighter do highlighting, even though its using the single frag list builder diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java index 72b819bd5b8e9..65cbb22312f4c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceScoreOrderFragmentsBuilder.java @@ -51,7 +51,7 @@ public SourceScoreOrderFragmentsBuilder(FieldMapper mapper, SearchContext search lookup.setNextReader(reader); lookup.setNextDocId(docId); - List values = lookup.source().getValues(mapper.names().fullName()); + List values = lookup.source().extractRawValues(mapper.names().fullName()); Field[] fields = new Field[values.size()]; for (int i = 0; i < values.size(); i++) { fields[i] = new Field(mapper.names().indexName(), values.get(i).toString(), Field.Store.NO, Field.Index.ANALYZED); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java index 76b5534676a62..819883ab37c53 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/highlight/vectorhighlight/SourceSimpleFragmentsBuilder.java @@ -53,7 +53,7 @@ public SourceSimpleFragmentsBuilder(FieldMapper mapper, SearchContext searchCont lookup.setNextReader(reader); lookup.setNextDocId(docId); - List values = lookup.source().getValues(mapper.names().fullName()); + List values = lookup.source().extractRawValues(mapper.names().fullName()); if (values.isEmpty()) { return EMPTY_FIELDS; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index 03b269e17ced3..feb309b9f2fd5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -23,7 +23,6 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; import org.elasticsearch.ElasticSearchParseException; -import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.compress.lzf.LZF; import org.elasticsearch.common.io.stream.BytesStreamInput; import org.elasticsearch.common.io.stream.CachedStreamInput; @@ -31,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.internal.SourceFieldMapper; import org.elasticsearch.index.mapper.internal.SourceFieldSelector; @@ -126,46 +126,12 @@ public void setNextSource(Map source) { * Returns the values associated with the path. Those are "low" level values, and it can * handle path expression where an array/list is navigated within. */ - public List getValues(String path) { - List values = Lists.newArrayList(); - String[] pathElements = dotPattern.split(path); - getValues(values, loadSourceIfNeeded(), pathElements, 0); - return values; + public List extractRawValues(String path) { + return XContentMapValues.extractRawValues(path, loadSourceIfNeeded()); } - @SuppressWarnings({"unchecked"}) - private void getValues(List values, Map part, String[] pathElements, int index) { - if (index == pathElements.length) { - return; - } - String currentPath = pathElements[index]; - Object currentValue = part.get(currentPath); - if (currentValue == null) { - return; - } - if (currentValue instanceof Map) { - getValues(values, (Map) currentValue, pathElements, index + 1); - } else if (currentValue instanceof List) { - getValues(values, (List) currentValue, pathElements, index + 1); - } else { - values.add(currentValue); - } - } - - @SuppressWarnings({"unchecked"}) - private void getValues(List values, List part, String[] pathElements, int index) { - for (Object value : part) { - if (value == null) { - continue; - } - if (value instanceof Map) { - getValues(values, (Map) value, pathElements, index); - } else if (value instanceof List) { - getValues(values, (List) value, pathElements, index); - } else { - values.add(value); - } - } + public Object extractValue(String path) { + return XContentMapValues.extractValue(path, loadSourceIfNeeded()); } @Override public Object get(Object key) { diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java new file mode 100644 index 0000000000000..4ac2b06bf773c --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java @@ -0,0 +1,99 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.xcontent.support; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.testng.annotations.Test; + +import java.util.List; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + */ +@Test +public class XContentMapValuesTests { + + @SuppressWarnings({"unchecked"}) @Test public void testExtractValue() throws Exception { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject() + .field("test", "value") + .endObject(); + + Map map = XContentFactory.xContent(XContentType.JSON).createParser(builder.string()).mapAndClose(); + assertThat(XContentMapValues.extractValue("test", map).toString(), equalTo("value")); + assertThat(XContentMapValues.extractValue("test.me", map), nullValue()); + assertThat(XContentMapValues.extractValue("something.else.2", map), nullValue()); + + builder = XContentFactory.jsonBuilder().startObject() + .startObject("path1").startObject("path2").field("test", "value").endObject().endObject() + .endObject(); + + map = XContentFactory.xContent(XContentType.JSON).createParser(builder.string()).mapAndClose(); + assertThat(XContentMapValues.extractValue("path1.path2.test", map).toString(), equalTo("value")); + assertThat(XContentMapValues.extractValue("path1.path2.test_me", map), nullValue()); + assertThat(XContentMapValues.extractValue("path1.non_path2.test", map), nullValue()); + + Object extValue = XContentMapValues.extractValue("path1.path2", map); + assertThat(extValue, instanceOf(Map.class)); + Map extMapValue = (Map) extValue; + assertThat(extMapValue, hasEntry("test", (Object) "value")); + + extValue = XContentMapValues.extractValue("path1", map); + assertThat(extValue, instanceOf(Map.class)); + extMapValue = (Map) extValue; + assertThat(extMapValue.containsKey("path2"), equalTo(true)); + + // lists + builder = XContentFactory.jsonBuilder().startObject() + .startObject("path1").field("test", "value1", "value2").endObject() + .endObject(); + + map = XContentFactory.xContent(XContentType.JSON).createParser(builder.string()).mapAndClose(); + + extValue = XContentMapValues.extractValue("path1.test", map); + assertThat(extValue, instanceOf(List.class)); + + List extListValue = (List) extValue; + assertThat(extListValue.size(), equalTo(2)); + + builder = XContentFactory.jsonBuilder().startObject() + .startObject("path1") + .startArray("path2") + .startObject().field("test", "value1").endObject() + .startObject().field("test", "value2").endObject() + .endArray() + .endObject() + .endObject(); + + map = XContentFactory.xContent(XContentType.JSON).createParser(builder.string()).mapAndClose(); + + extValue = XContentMapValues.extractValue("path1.path2.test", map); + assertThat(extValue, instanceOf(List.class)); + + extListValue = (List) extValue; + assertThat(extListValue.size(), equalTo(2)); + assertThat(extListValue.get(0).toString(), equalTo("value1")); + assertThat(extListValue.get(1).toString(), equalTo("value2")); + } +} diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java index 0f3890a0db17c..2357496b4b4a0 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/fields/SearchFieldsTests.java @@ -66,8 +66,6 @@ protected Client getClient() { client.admin().indices().preparePutMapping().setType("type1").setSource(mapping).execute().actionGet(); - Thread.sleep(100); // sleep a bit here..., so hte mappings get applied - client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() .field("field1", "value1") .field("field2", "value2") From 8c322b4cc26412b2accc236763dedbe9363bab88 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 20 Sep 2011 16:54:02 +0300 Subject: [PATCH 39/96] use the same improved fields extraction in get api --- .../index/get/ShardGetService.java | 66 ++++++++++--------- .../search/lookup/SourceLookup.java | 3 - 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java index f9318b4843752..a0777f6f6dd56 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java @@ -189,40 +189,45 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real if (gFields != null && gFields.length > 0) { SearchLookup searchLookup = null; for (String field : gFields) { - String script = null; + Object value = null; if (field.contains("_source.") || field.contains("doc[")) { - script = field; - } else { - FieldMappers x = docMapper.mappers().smartName(field); - if (x != null && !x.mapper().stored()) { - script = "_source." + x.mapper().names().fullName(); - } - } - if (script != null) { if (searchLookup == null) { searchLookup = new SearchLookup(mapperService, indexCache.fieldData()); } - SearchScript searchScript = scriptService.search(searchLookup, "mvel", script, null); + SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null); searchScript.setNextReader(docIdAndVersion.reader); searchScript.setNextDocId(docIdAndVersion.docId); try { - Object value = searchScript.run(); - if (fields == null) { - fields = newHashMapWithExpectedSize(2); - } - GetField getField = fields.get(field); - if (getField == null) { - getField = new GetField(field, new ArrayList(2)); - fields.put(field, getField); - } - getField.values().add(value); + value = searchScript.run(); } catch (RuntimeException e) { if (logger.isTraceEnabled()) { - logger.trace("failed to execute get request script field [{}]", e, script); + logger.trace("failed to execute get request script field [{}]", e, field); } // ignore } + } else { + FieldMappers x = docMapper.mappers().smartName(field); + if (x == null || !x.mapper().stored()) { + if (searchLookup == null) { + searchLookup = new SearchLookup(mapperService, indexCache.fieldData()); + searchLookup.setNextReader(docIdAndVersion.reader); + searchLookup.setNextDocId(docIdAndVersion.docId); + } + value = searchLookup.source().extractValue(field); + } + } + + if (value != null) { + if (fields == null) { + fields = newHashMapWithExpectedSize(2); + } + GetField getField = fields.get(field); + if (getField == null) { + getField = new GetField(field, new ArrayList(2)); + fields.put(field, getField); + } + getField.values().add(value); } } } @@ -261,23 +266,14 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real value = docMapper.TTLFieldMapper().valueForSearch(source.timestamp + source.ttl); } } else { - String script = null; if (field.contains("_source.")) { - script = field; - } else { - FieldMappers x = docMapper.mappers().smartName(field); - if (x != null) { - script = "_source." + x.mapper().names().fullName(); - } - } - if (script != null) { if (searchLookup == null) { searchLookup = new SearchLookup(mapperService, indexCache.fieldData()); } if (sourceAsMap == null) { sourceAsMap = SourceLookup.sourceAsMap(source.source.bytes(), source.source.offset(), source.source.length()); } - SearchScript searchScript = scriptService.search(searchLookup, "mvel", script, null); + SearchScript searchScript = scriptService.search(searchLookup, "mvel", field, null); // we can't do this, only allow to run scripts against the source //searchScript.setNextReader(docIdAndVersion.reader); //searchScript.setNextDocId(docIdAndVersion.docId); @@ -289,10 +285,16 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real value = searchScript.run(); } catch (RuntimeException e) { if (logger.isTraceEnabled()) { - logger.trace("failed to execute get request script field [{}]", e, script); + logger.trace("failed to execute get request script field [{}]", e, field); } // ignore } + } else { + if (searchLookup == null) { + searchLookup = new SearchLookup(mapperService, indexCache.fieldData()); + searchLookup.source().setNextSource(SourceLookup.sourceAsMap(source.source.bytes(), source.source.offset(), source.source.length())); + } + value = searchLookup.source().extractValue(field); } } if (value != null) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index feb309b9f2fd5..9d223012a09f2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -38,7 +38,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.regex.Pattern; /** * @author kimchy (shay.banon) @@ -120,8 +119,6 @@ public void setNextSource(Map source) { this.source = source; } - private final static Pattern dotPattern = Pattern.compile("\\."); - /** * Returns the values associated with the path. Those are "low" level values, and it can * handle path expression where an array/list is navigated within. From 6a6cba1ff3426d9d62ec90ac01579e17a2d003a8 Mon Sep 17 00:00:00 2001 From: Njal Karevoll Date: Tue, 20 Sep 2011 16:37:14 +0200 Subject: [PATCH 40/96] include the path when serializing _id field mappings --- .../index/mapper/internal/IdFieldMapper.java | 5 ++++- .../index/mapper/id/IdMappingTests.java | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 1199aed7ccf56..af2c2d4e24edf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -198,7 +198,7 @@ public String value(Document document) { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { // if all are defaults, no sense to write it at all - if (store == Defaults.STORE && index == Defaults.INDEX) { + if (store == Defaults.STORE && index == Defaults.INDEX && path == Defaults.PATH) { return builder; } builder.startObject(CONTENT_TYPE); @@ -208,6 +208,9 @@ public String value(Document document) { if (index != Defaults.INDEX) { builder.field("index", index.name().toLowerCase()); } + if (path != Defaults.PATH) { + builder.field("path", path); + } builder.endObject(); return builder; } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java index ef529dc3a6722..7f40cadd45cd3 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.mapper.id; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; @@ -91,4 +93,23 @@ public class IdMappingTests { assertThat(doc.rootDoc().get(UidFieldMapper.NAME), notNullValue()); assertThat(doc.rootDoc().get(IdFieldMapper.NAME), notNullValue()); } + + @Test public void testIdPath() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_id").field("path", "my_path").endObject() + .endObject().endObject().string(); + DocumentMapper docMapper = MapperTests.newParser().parse(mapping); + + // serialize the id mapping + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder = docMapper.idFieldMapper().toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + String serialized_id_mapping = builder.string(); + + String expected_id_mapping = XContentFactory.jsonBuilder().startObject() + .startObject("_id").field("path", "my_path").endObject() + .endObject().string(); + + assertThat(serialized_id_mapping, equalTo(expected_id_mapping)); + } } From 3028d5a7a16cd9ccb28f1a364ac2c1d762a6e6de Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Sep 2011 18:26:16 +0300 Subject: [PATCH 41/96] Shard allocation awareness (rack aware, zone aware, for example), closes #1352. --- .../cluster/routing/RoutingNodes.java | 18 + .../routing/allocation/AllocationService.java | 41 +- .../decider/AllocationDeciders.java | 1 + .../decider/AllocationDecidersModule.java | 1 + .../decider/AwarenessAllocationDecider.java | 142 ++++ .../common/settings/ImmutableSettings.java | 6 +- .../allocation/AwarenessAllocationTests.java | 727 ++++++++++++++++++ .../allocation/FilterRoutingTests.java | 2 +- 8 files changed, 923 insertions(+), 15 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index a7680da5bc00a..6ee83ab92d396 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -23,11 +23,13 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; import org.elasticsearch.common.util.concurrent.NotThreadSafe; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -53,6 +55,8 @@ public class RoutingNodes implements Iterable { private final List ignoredUnassigned = newArrayList(); + private final Map> nodesPerAttributeNames = new HashMap>(); + public RoutingNodes(ClusterState clusterState) { this.metaData = clusterState.metaData(); this.blocks = clusterState.blocks(); @@ -158,6 +162,20 @@ public RoutingNode node(String nodeId) { return nodesToShards.get(nodeId); } + public TObjectIntHashMap nodesPerAttributesCounts(String attributeName) { + TObjectIntHashMap nodesPerAttributesCounts = nodesPerAttributeNames.get(attributeName); + if (nodesPerAttributesCounts != null) { + return nodesPerAttributesCounts; + } + nodesPerAttributesCounts = new TObjectIntHashMap(); + for (RoutingNode routingNode : this) { + String attrValue = routingNode.node().attributes().get(attributeName); + nodesPerAttributesCounts.adjustOrPutValue(attrValue, 1, 1); + } + nodesPerAttributeNames.put(attributeName, nodesPerAttributesCounts); + return nodesPerAttributesCounts; + } + public MutableShardRouting findPrimaryForReplica(ShardRouting shard) { assert !shard.primary(); for (RoutingNode routingNode : nodesToShards.values()) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index dfe7c6f496f1c..5b9c21d35fd15 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.settings.NodeSettingsService; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -175,20 +176,36 @@ private boolean reroute(RoutingAllocation allocation) { private boolean moveShards(RoutingAllocation allocation) { boolean changed = false; - for (RoutingNode routingNode : allocation.routingNodes()) { - for (MutableShardRouting shardRouting : routingNode) { - // we can only move started shards... - if (!shardRouting.started()) { + + // create a copy of the shards interleaving between nodes, and check if they can remain + List shards = new ArrayList(); + int index = 0; + boolean found = true; + while (found) { + found = false; + for (RoutingNode routingNode : allocation.routingNodes()) { + if (index >= routingNode.shards().size()) { continue; } - if (!allocation.deciders().canRemain(shardRouting, routingNode, allocation)) { - logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node()); - boolean moved = shardsAllocators.move(shardRouting, routingNode, allocation); - if (!moved) { - logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id()); - } else { - changed = true; - } + found = true; + shards.add(routingNode.shards().get(index)); + } + index++; + } + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shardRouting = shards.get(i); + // we can only move started shards... + if (!shardRouting.started()) { + continue; + } + RoutingNode routingNode = allocation.routingNodes().node(shardRouting.currentNodeId()); + if (!allocation.deciders().canRemain(shardRouting, routingNode, allocation)) { + logger.debug("[{}][{}] allocated on [{}], but can no longer be allocated on it, moving...", shardRouting.index(), shardRouting.id(), routingNode.node()); + boolean moved = shardsAllocators.move(shardRouting, routingNode, allocation); + if (!moved) { + logger.debug("[{}][{}] can't move", shardRouting.index(), shardRouting.id()); + } else { + changed = true; } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index f157da7673514..704e9226fb149 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -45,6 +45,7 @@ public AllocationDeciders(Settings settings, NodeSettingsService nodeSettingsSer .add(new RebalanceOnlyWhenActiveAllocationDecider(settings)) .add(new ClusterRebalanceAllocationDecider(settings)) .add(new ConcurrentRebalanceAllocationDecider(settings)) + .add(new AwarenessAllocationDecider(settings)) .build() ); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java index be318f9eca411..675fbe8bf1faf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java @@ -47,6 +47,7 @@ public AllocationDecidersModule(Settings settings) { allocationMultibinder.addBinding().to(RebalanceOnlyWhenActiveAllocationDecider.class); allocationMultibinder.addBinding().to(ClusterRebalanceAllocationDecider.class); allocationMultibinder.addBinding().to(ConcurrentRebalanceAllocationDecider.class); + allocationMultibinder.addBinding().to(AwarenessAllocationDecider.class); for (Class allocation : allocations) { allocationMultibinder.addBinding().to(allocation); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java new file mode 100644 index 0000000000000..75c7680803594 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -0,0 +1,142 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.MutableShardRouting; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.collect.Maps; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; + +import java.util.Map; + +/** + */ +public class AwarenessAllocationDecider extends AllocationDecider { + + private String[] awarenessAttributes; + + private Map forcedAwarenessAttributes; + + @Inject public AwarenessAllocationDecider(Settings settings) { + super(settings); + this.awarenessAttributes = settings.getAsArray("cluster.routing.allocation.awareness.attributes"); + + forcedAwarenessAttributes = Maps.newHashMap(); + Map forceGroups = settings.getGroups("cluster.routing.allocation.awareness.force."); + for (Map.Entry entry : forceGroups.entrySet()) { + forcedAwarenessAttributes.put(entry.getKey(), entry.getValue().getAsArray("values")); + } + } + + @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return underCapacity(shardRouting, node, allocation, true) ? Decision.YES : Decision.NO; + } + + @Override public boolean canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + return underCapacity(shardRouting, node, allocation, false); + } + + private boolean underCapacity(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation, boolean moveToNode) { + if (awarenessAttributes.length == 0) { + return true; + } + + IndexMetaData indexMetaData = allocation.metaData().index(shardRouting.index()); + int shardCount = indexMetaData.numberOfReplicas() + 1; // 1 for primary + for (String awarenessAttribute : awarenessAttributes) { + // the node the shard exists on must be associated with an awareness attribute + if (!node.node().attributes().containsKey(awarenessAttribute)) { + return false; + } + + // build attr_value -> nodes map + TObjectIntHashMap nodesPerAttribute = allocation.routingNodes().nodesPerAttributesCounts(awarenessAttribute); + + // build the count of shards per attribute value + TObjectIntHashMap shardPerAttribute = new TObjectIntHashMap(); + for (RoutingNode routingNode : allocation.routingNodes()) { + for (int i = 0; i < routingNode.shards().size(); i++) { + MutableShardRouting nodeShardRouting = routingNode.shards().get(i); + if (nodeShardRouting.shardId().equals(shardRouting.shardId())) { + // if the shard is relocating, then make sure we count it as part of the node it is relocating to + if (nodeShardRouting.relocating()) { + RoutingNode relocationNode = allocation.routingNodes().node(nodeShardRouting.relocatingNodeId()); + shardPerAttribute.adjustOrPutValue(relocationNode.node().attributes().get(awarenessAttribute), 1, 1); + } else if (nodeShardRouting.started()) { + shardPerAttribute.adjustOrPutValue(routingNode.node().attributes().get(awarenessAttribute), 1, 1); + } + } + } + } + if (moveToNode) { + if (shardRouting.assignedToNode()) { + String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId(); + if (!node.nodeId().equals(nodeId)) { + // we work on different nodes, move counts around + shardPerAttribute.adjustOrPutValue(allocation.routingNodes().node(nodeId).node().attributes().get(awarenessAttribute), -1, 0); + shardPerAttribute.adjustOrPutValue(node.node().attributes().get(awarenessAttribute), 1, 1); + } + } else { + shardPerAttribute.adjustOrPutValue(node.node().attributes().get(awarenessAttribute), 1, 1); + } + } + + int numberOfAttributes = nodesPerAttribute.size(); + String[] fullValues = forcedAwarenessAttributes.get(awarenessAttribute); + if (fullValues != null) { + for (String fullValue : fullValues) { + if (!shardPerAttribute.contains(fullValue)) { + numberOfAttributes++; + } + } + } + // TODO should we remove ones that are not part of full list? + + int averagePerAttribute = shardCount / numberOfAttributes; + int totalLeftover = shardCount % numberOfAttributes; + int requiredCountPerAttribute; + if (averagePerAttribute == 0) { + // if we have more attributes values than shard count, no leftover + totalLeftover = 0; + requiredCountPerAttribute = 1; + } else { + requiredCountPerAttribute = averagePerAttribute; + } + int leftoverPerAttribute = totalLeftover == 0 ? 0 : 1; + + int currentNodeCount = shardPerAttribute.get(node.node().attributes().get(awarenessAttribute)); + // if we are above with leftover, then we know we are not good, even with mod + if (currentNodeCount > (requiredCountPerAttribute + leftoverPerAttribute)) { + return false; + } + // all is well, we are below or same as average + if (currentNodeCount <= requiredCountPerAttribute) { + continue; + } + } + + return true; + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index f2c0073775f6e..d259c5dad61a7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -237,7 +237,9 @@ private ImmutableSettings(Map settings, ClassLoader classLoader) if (get(settingPrefix) != null) { String[] strings = Strings.splitStringByCommaToArray(get(settingPrefix)); if (strings.length > 0) { - Collections.addAll(result, strings); + for (String string : strings) { + result.add(string.trim()); + } } } @@ -247,7 +249,7 @@ private ImmutableSettings(Map settings, ClassLoader classLoader) if (value == null) { break; } - result.add(value); + result.add(value.trim()); } if (result.isEmpty()) { return defaultArray; diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java new file mode 100644 index 0000000000000..56304c4230186 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/AwarenessAllocationTests.java @@ -0,0 +1,727 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.testng.annotations.Test; + +import static org.elasticsearch.cluster.ClusterState.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.MetaData.*; +import static org.elasticsearch.cluster.node.DiscoveryNodes.*; +import static org.elasticsearch.cluster.routing.RoutingBuilders.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.allocation.RoutingAllocationTests.*; +import static org.elasticsearch.common.settings.ImmutableSettings.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + */ +@Test +public class AwarenessAllocationTests { + + private final ESLogger logger = Loggers.getLogger(AwarenessAllocationTests.class); + + @Test public void moveShardOnceNewNodeWithAttributeAdded1() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node3")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, make sure nothing moves"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + assertThat(routingTable, sameInstance(clusterState.routingTable())); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(2)); + } + + @Test public void moveShardOnceNewNodeWithAttributeAdded2() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node3", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node4")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, make sure nothing moves"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node5", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + assertThat(routingTable, sameInstance(clusterState.routingTable())); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(2)); + } + + @Test public void moveShardOnceNewNodeWithAttributeAdded3() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.node_concurrent_recoveries", 10) + .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(5).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(5)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(5)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(5)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(5)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node3")); + + logger.info("--> complete initializing"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> run it again, since we still might have relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, some more relocation should happen"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + } + + @Test public void moveShardOnceNewNodeWithAttributeAdded4() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.node_concurrent_recoveries", 10) + .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test1").numberOfShards(5).numberOfReplicas(1)) + .put(newIndexMetaDataBuilder("test2").numberOfShards(5).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test1").initializeEmpty(metaData.index("test1"))) + .add(indexRoutingTable("test2").initializeEmpty(metaData.index("test2"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(10)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(10)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(10)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node3")); + + logger.info("--> complete initializing"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> run it again, since we still might have relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, some more relocation should happen"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + } + + @Test public void moveShardOnceNewNodeWithAttributeAdded5() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(2)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, we will have another relocation"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(2)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node4")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); + + logger.info("--> make sure another reroute does not move things"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + } + + @Test public void moveShardOnceNewNodeWithAttributeAdded6() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(3)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node3", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node4", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> start the shards (replicas)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node5", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(3)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node5")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, we will have another relocation"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node6", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(3)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.RELOCATING).get(0).relocatingNodeId(), equalTo("node6")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(4)); + + logger.info("--> make sure another reroute does not move things"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + } + + @Test public void fullAwareness1() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.force.rack_id.values", "1,2") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> replica will not start because we have only one rack value"); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, make sure nothing moves"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + assertThat(routingTable, sameInstance(clusterState.routingTable())); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(2)); + } + + @Test public void fullAwareness2() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.force.rack_id.values", "1,2") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node3", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> replica will not start because we have only one rack value"); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node4")); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(2)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, make sure nothing moves"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node5", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + assertThat(routingTable, sameInstance(clusterState.routingTable())); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(STARTED).size(), equalTo(2)); + } + + @Test public void fullAwareness3() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.node_concurrent_recoveries", 10) + .put("cluster.routing.allocation.node_initial_primaries_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.cluster_concurrent_rebalance", -1) + .put("cluster.routing.allocation.awareness.force.rack_id.values", "1,2") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test1").numberOfShards(5).numberOfReplicas(1)) + .put(newIndexMetaDataBuilder("test2").numberOfShards(5).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test1").initializeEmpty(metaData.index("test1"))) + .add(indexRoutingTable("test2").initializeEmpty(metaData.index("test2"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "1"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(10)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + + logger.info("--> add a new node with a new rack and reroute"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node3", ImmutableMap.of("rack_id", "2"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(10)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(10)); + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node3")); + + logger.info("--> complete initializing"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + logger.info("--> run it again, since we still might have relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + + logger.info("--> add another node with a new rack, some more relocation should happen"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder().putAll(clusterState.nodes()) + .put(newNode("node4", ImmutableMap.of("rack_id", "3"))) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(RELOCATING).size(), greaterThan(0)); + + logger.info("--> complete relocation"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(20)); + + logger.info("--> do another reroute, make sure nothing moves"); + assertThat(strategy.reroute(clusterState).routingTable(), sameInstance(clusterState.routingTable())); + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java index c83a446fe56b1..756164bf6c57e 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/FilterRoutingTests.java @@ -68,7 +68,7 @@ public class FilterRoutingTests { ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); - logger.info("--> adding two nodes and performing rerouting"); + logger.info("--> adding four nodes and performing rerouting"); clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() .put(newNode("node1", ImmutableMap.of("tag1", "value1"))) .put(newNode("node2", ImmutableMap.of("tag1", "value2"))) From 44efcca108a09676c8f570d9c0830a00202c9840 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Sep 2011 23:09:44 +0300 Subject: [PATCH 42/96] automatic preference when using awareness attributes --- .../routing/IndexShardRoutingTable.java | 86 +++++++++++++++++++ .../decider/AwarenessAllocationDecider.java | 4 + .../plain/PlainOperationRouting.java | 33 +++++-- .../structure/RoutingIteratorTests.java | 57 ++++++++++++ ...dsUnbalancedShardsEmbeddedSearchTests.java | 3 +- 5 files changed, 173 insertions(+), 10 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java index 27f9b68773e71..dadd92b9addaf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/IndexShardRoutingTable.java @@ -19,7 +19,10 @@ package org.elasticsearch.cluster.routing; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.collect.UnmodifiableIterator; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -28,7 +31,10 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.collect.Lists.*; @@ -264,6 +270,86 @@ private ShardIterator preferNodeShardsIt(String nodeId, ImmutableList withSameAttribute; + public final ImmutableList withoutSameAttribute; + public final int totalSize; + + AttributesRoutings(ImmutableList withSameAttribute, ImmutableList withoutSameAttribute) { + this.withSameAttribute = withSameAttribute; + this.withoutSameAttribute = withoutSameAttribute; + this.totalSize = withoutSameAttribute.size() + withSameAttribute.size(); + } + } + + private volatile Map activeShardsByAttributes = ImmutableMap.of(); + private final Object shardsByAttributeMutex = new Object(); + + public ShardIterator preferAttributesActiveShardsIt(String[] attributes, DiscoveryNodes nodes) { + return preferAttributesActiveShardsIt(attributes, nodes, counter.incrementAndGet()); + } + + public ShardIterator preferAttributesActiveShardsIt(String[] attributes, DiscoveryNodes nodes, int index) { + AttributesKey key = new AttributesKey(attributes); + AttributesRoutings shardRoutings = activeShardsByAttributes.get(key); + if (shardRoutings == null) { + synchronized (shardsByAttributeMutex) { + ArrayList from = new ArrayList(activeShards); + ArrayList to = new ArrayList(); + for (String attribute : attributes) { + String localAttributeValue = nodes.localNode().attributes().get(attribute); + if (localAttributeValue == null) { + continue; + } + for (Iterator iterator = from.iterator(); iterator.hasNext(); ) { + ShardRouting fromShard = iterator.next(); + if (localAttributeValue.equals(nodes.get(fromShard.currentNodeId()).attributes().get(attribute))) { + iterator.remove(); + to.add(fromShard); + } + } + } + + shardRoutings = new AttributesRoutings(ImmutableList.copyOf(to), ImmutableList.copyOf(from)); + activeShardsByAttributes = MapBuilder.newMapBuilder(activeShardsByAttributes).put(key, shardRoutings).immutableMap(); + } + } + // we now randomize, once between the ones that have the same attributes, and once for the ones that don't + // we don't want to mix between the two! + ArrayList ordered = new ArrayList(shardRoutings.totalSize); + index = Math.abs(index); + for (int i = 0; i < shardRoutings.withSameAttribute.size(); i++) { + int loc = (index + i) % shardRoutings.withSameAttribute.size(); + ShardRouting shardRouting = shardRoutings.withSameAttribute.get(loc); + ordered.add(shardRouting); + } + for (int i = 0; i < shardRoutings.withoutSameAttribute.size(); i++) { + int loc = (index + i) % shardRoutings.withoutSameAttribute.size(); + ShardRouting shardRouting = shardRoutings.withoutSameAttribute.get(loc); + ordered.add(shardRouting); + } + + return new PlainShardIterator(shardId, ordered); + } + public ShardRouting primaryShard() { return primary; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 75c7680803594..6536c489764cb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -50,6 +50,10 @@ public class AwarenessAllocationDecider extends AllocationDecider { } } + public String[] awarenessAttributes() { + return this.awarenessAttributes; + } + @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { return underCapacity(shardRouting, node, allocation, true) ? Decision.YES : Decision.NO; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/operation/plain/PlainOperationRouting.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/operation/plain/PlainOperationRouting.java index a8e22df022685..cb2225700ec50 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/operation/plain/PlainOperationRouting.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/operation/plain/PlainOperationRouting.java @@ -22,10 +22,12 @@ import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.GroupShardsIterator; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.IndexShardRoutingTable; import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.elasticsearch.cluster.routing.operation.OperationRouting; import org.elasticsearch.cluster.routing.operation.hash.HashFunction; import org.elasticsearch.cluster.routing.operation.hash.djb.DjbHashFunction; @@ -52,10 +54,13 @@ public class PlainOperationRouting extends AbstractComponent implements Operatio private final boolean useType; - @Inject public PlainOperationRouting(Settings indexSettings, HashFunction hashFunction) { + private final AwarenessAllocationDecider awarenessAllocationDecider; + + @Inject public PlainOperationRouting(Settings indexSettings, HashFunction hashFunction, AwarenessAllocationDecider awarenessAllocationDecider) { super(indexSettings); this.hashFunction = hashFunction; this.useType = indexSettings.getAsBoolean("cluster.routing.operation.use_type", false); + this.awarenessAllocationDecider = awarenessAllocationDecider; } @Override public ShardIterator indexShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing) throws IndexMissingException, IndexShardMissingException { @@ -67,11 +72,11 @@ public class PlainOperationRouting extends AbstractComponent implements Operatio } @Override public ShardIterator getShards(ClusterState clusterState, String index, String type, String id, @Nullable String routing, @Nullable String preference) throws IndexMissingException, IndexShardMissingException { - return preferenceActiveShardIterator(shards(clusterState, index, type, id, routing), clusterState.nodes().localNodeId(), preference); + return preferenceActiveShardIterator(shards(clusterState, index, type, id, routing), clusterState.nodes().localNodeId(), clusterState.nodes(), preference); } @Override public ShardIterator getShards(ClusterState clusterState, String index, int shardId, @Nullable String preference) throws IndexMissingException, IndexShardMissingException { - return preferenceActiveShardIterator(shards(clusterState, index, shardId), clusterState.nodes().localNodeId(), preference); + return preferenceActiveShardIterator(shards(clusterState, index, shardId), clusterState.nodes().localNodeId(), clusterState.nodes(), preference); } @Override public GroupShardsIterator broadcastDeleteShards(ClusterState clusterState, String index) throws IndexMissingException { @@ -149,7 +154,7 @@ public class PlainOperationRouting extends AbstractComponent implements Operatio throw new IndexShardMissingException(new ShardId(index, shardId)); } // we might get duplicates, but that's ok, they will override one another - set.add(preferenceActiveShardIterator(indexShard, clusterState.nodes().localNodeId(), preference)); + set.add(preferenceActiveShardIterator(indexShard, clusterState.nodes().localNodeId(), clusterState.nodes(), preference)); } } } @@ -160,25 +165,35 @@ public class PlainOperationRouting extends AbstractComponent implements Operatio for (String index : concreteIndices) { IndexRoutingTable indexRouting = indexRoutingTable(clusterState, index); for (IndexShardRoutingTable indexShard : indexRouting) { - set.add(preferenceActiveShardIterator(indexShard, clusterState.nodes().localNodeId(), preference)); + set.add(preferenceActiveShardIterator(indexShard, clusterState.nodes().localNodeId(), clusterState.nodes(), preference)); } } return new GroupShardsIterator(set); } } - private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable indexShard, String nodeId, @Nullable String preference) { + private ShardIterator preferenceActiveShardIterator(IndexShardRoutingTable indexShard, String nodeId, DiscoveryNodes nodes, @Nullable String preference) { if (preference == null) { - return indexShard.activeShardsRandomIt(); + String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes(); + if (awarenessAttributes.length == 0) { + return indexShard.activeShardsRandomIt(); + } else { + return indexShard.preferAttributesActiveShardsIt(awarenessAttributes, nodes); + } } if ("_local".equals(preference)) { - return indexShard.preferNodeShardsIt(nodeId); + return indexShard.preferNodeActiveShardsIt(nodeId); } if ("_primary".equals(preference)) { return indexShard.primaryShardIt(); } // if not, then use it as the index - return indexShard.shardsIt(DjbHashFunction.DJB_HASH(preference)); + String[] awarenessAttributes = awarenessAllocationDecider.awarenessAttributes(); + if (awarenessAttributes.length == 0) { + return indexShard.activeShardsIt(DjbHashFunction.DJB_HASH(preference)); + } else { + return indexShard.preferAttributesActiveShardsIt(awarenessAttributes, nodes, DjbHashFunction.DJB_HASH(preference)); + } } public IndexMetaData indexMetaData(ClusterState clusterState, String index) { diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java index dbbc2acd52f08..576c26350e6a2 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/structure/RoutingIteratorTests.java @@ -19,18 +19,26 @@ package org.elasticsearch.cluster.structure; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.PlainShardIterator; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.index.shard.ShardId; import org.testng.annotations.Test; +import static org.elasticsearch.cluster.ClusterState.*; import static org.elasticsearch.cluster.metadata.IndexMetaData.*; import static org.elasticsearch.cluster.metadata.MetaData.*; +import static org.elasticsearch.cluster.node.DiscoveryNodes.*; import static org.elasticsearch.cluster.routing.RoutingBuilders.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.allocation.RoutingAllocationTests.*; +import static org.elasticsearch.common.settings.ImmutableSettings.*; import static org.hamcrest.MatcherAssert.*; import static org.hamcrest.Matchers.*; @@ -209,4 +217,53 @@ public class RoutingIteratorTests { assertThat(shardRouting1, not(sameInstance(shardRouting2))); assertThat(shardRouting1, sameInstance(shardRouting3)); } + + @Test public void testAttributePreferenceRouting() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.concurrent_recoveries", 10) + .put("cluster.routing.allocation.allow_rebalance", "always") + .put("cluster.routing.allocation.awareness.attributes", "rack_id,zone") + .build()); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1", ImmutableMap.of("rack_id", "rack_1", "zone", "zone1"))) + .put(newNode("node2", ImmutableMap.of("rack_id", "rack_2", "zone", "zone2"))) + .localNodeId("node1") + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + // after all are started, check routing iteration + ShardIterator shardIterator = clusterState.routingTable().index("test").shard(0).preferAttributesActiveShardsIt(new String[]{"rack_id"}, clusterState.nodes()); + ShardRouting shardRouting = shardIterator.nextOrNull(); + assertThat(shardRouting, notNullValue()); + assertThat(shardRouting.currentNodeId(), equalTo("node1")); + shardRouting = shardIterator.nextOrNull(); + assertThat(shardRouting, notNullValue()); + assertThat(shardRouting.currentNodeId(), equalTo("node2")); + + shardIterator = clusterState.routingTable().index("test").shard(0).preferAttributesActiveShardsIt(new String[]{"rack_id"}, clusterState.nodes()); + shardRouting = shardIterator.nextOrNull(); + assertThat(shardRouting, notNullValue()); + assertThat(shardRouting.currentNodeId(), equalTo("node1")); + shardRouting = shardIterator.nextOrNull(); + assertThat(shardRouting, notNullValue()); + assertThat(shardRouting.currentNodeId(), equalTo("node2")); + } } \ No newline at end of file diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java index b6b06f7ffdbaa..add2c04326e01 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterService; import org.elasticsearch.cluster.routing.ShardIterator; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.elasticsearch.cluster.routing.operation.OperationRouting; import org.elasticsearch.cluster.routing.operation.plain.PlainOperationRouting; import org.elasticsearch.common.collect.ImmutableMap; @@ -399,7 +400,7 @@ public static class UnevenOperationRoutingModule extends AbstractModule { public static class UnevenOperationRoutingStrategy extends PlainOperationRouting { @Inject public UnevenOperationRoutingStrategy(Settings settings) { - super(settings, null); + super(settings, null, new AwarenessAllocationDecider(Builder.EMPTY_SETTINGS)); } @Override protected int hash(String routing) { From 3f8b7f0fce96727beea3696fd70897d058b1a65b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 21 Sep 2011 23:48:23 +0300 Subject: [PATCH 43/96] allow to change awareness settings using cluster update settings API --- .../decider/AllocationDeciders.java | 2 +- .../decider/AwarenessAllocationDecider.java | 40 ++++++++++++++++++- .../cluster/MinimumMasterNodesTests.java | 2 +- ...dsUnbalancedShardsEmbeddedSearchTests.java | 3 +- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 704e9226fb149..551ed870dec9e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -45,7 +45,7 @@ public AllocationDeciders(Settings settings, NodeSettingsService nodeSettingsSer .add(new RebalanceOnlyWhenActiveAllocationDecider(settings)) .add(new ClusterRebalanceAllocationDecider(settings)) .add(new ConcurrentRebalanceAllocationDecider(settings)) - .add(new AwarenessAllocationDecider(settings)) + .add(new AwarenessAllocationDecider(settings, nodeSettingsService)) .build() ); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java index 6536c489764cb..85ad4248ac8a3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AwarenessAllocationDecider.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.MutableShardRouting; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -28,26 +29,61 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; +import org.elasticsearch.node.settings.NodeSettingsService; +import java.util.HashMap; import java.util.Map; /** */ public class AwarenessAllocationDecider extends AllocationDecider { + static { + MetaData.addDynamicSettings( + "cluster.routing.allocation.awareness.attributes", + "cluster.routing.allocation.awareness.force.*" + ); + } + + class ApplySettings implements NodeSettingsService.Listener { + @Override public void onRefreshSettings(Settings settings) { + String[] awarenessAttributes = settings.getAsArray("cluster.routing.allocation.awareness.attributes", null); + if (awarenessAttributes != null) { + logger.info("updating [cluster.routing.allocation.awareness.attributes] from [{}] to [{}]", AwarenessAllocationDecider.this.awarenessAttributes, awarenessAttributes); + AwarenessAllocationDecider.this.awarenessAttributes = awarenessAttributes; + } + Map forcedAwarenessAttributes = new HashMap(AwarenessAllocationDecider.this.forcedAwarenessAttributes); + Map forceGroups = settings.getGroups("cluster.routing.allocation.awareness.force."); + if (!forceGroups.isEmpty()) { + for (Map.Entry entry : forceGroups.entrySet()) { + String[] aValues = entry.getValue().getAsArray("values"); + if (aValues.length > 0) { + forcedAwarenessAttributes.put(entry.getKey(), aValues); + } + } + } + AwarenessAllocationDecider.this.forcedAwarenessAttributes = forcedAwarenessAttributes; + } + } + private String[] awarenessAttributes; private Map forcedAwarenessAttributes; - @Inject public AwarenessAllocationDecider(Settings settings) { + @Inject public AwarenessAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { super(settings); this.awarenessAttributes = settings.getAsArray("cluster.routing.allocation.awareness.attributes"); forcedAwarenessAttributes = Maps.newHashMap(); Map forceGroups = settings.getGroups("cluster.routing.allocation.awareness.force."); for (Map.Entry entry : forceGroups.entrySet()) { - forcedAwarenessAttributes.put(entry.getKey(), entry.getValue().getAsArray("values")); + String[] aValues = entry.getValue().getAsArray("values"); + if (aValues.length > 0) { + forcedAwarenessAttributes.put(entry.getKey(), aValues); + } } + + nodeSettingsService.addListener(new ApplySettings()); } public String[] awarenessAttributes() { diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java index 105faf8856caa..873d4a720655c 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/MinimumMasterNodesTests.java @@ -246,7 +246,7 @@ public class MinimumMasterNodesTests extends AbstractZenNodesTests { closeNode(nodeToShutdown); } - Thread.sleep(500); + Thread.sleep(1000); String lastNonMasterNodeUp = nonMasterNodes.removeLast(); logger.info("--> verify that there is no master anymore on remaining nodes"); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java index add2c04326e01..c1b56d9914fa4 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/embedded/ThreeShardsUnbalancedShardsEmbeddedSearchTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.trove.ExtTIntArrayList; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.internal.InternalNode; +import org.elasticsearch.node.settings.NodeSettingsService; import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -400,7 +401,7 @@ public static class UnevenOperationRoutingModule extends AbstractModule { public static class UnevenOperationRoutingStrategy extends PlainOperationRouting { @Inject public UnevenOperationRoutingStrategy(Settings settings) { - super(settings, null, new AwarenessAllocationDecider(Builder.EMPTY_SETTINGS)); + super(settings, null, new AwarenessAllocationDecider(Builder.EMPTY_SETTINGS, new NodeSettingsService(Builder.EMPTY_SETTINGS))); } @Override protected int hash(String routing) { From 41dbcdb7d67b1ba0feb93b6c6ad9ac72341ab14b Mon Sep 17 00:00:00 2001 From: Njal Karevoll Date: Tue, 20 Sep 2011 18:49:48 +0200 Subject: [PATCH 44/96] Give the date field mapping a "numeric_precision" argument that allows a user to configure the precision of the numeric timestamps. Supports all the time units from http://download.oracle.com/javase/6/docs/api/java/util/concurrent/TimeUnit.html --- .../index/mapper/core/DateFieldMapper.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 05184b74aa76c..6d8cc1af63faf 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -46,6 +46,7 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.index.mapper.MapperBuilders.*; import static org.elasticsearch.index.mapper.core.TypeParsers.*; @@ -61,10 +62,14 @@ public static class Defaults extends NumberFieldMapper.Defaults { public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime"); public static final String NULL_VALUE = null; + + public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS; } public static class Builder extends NumberFieldMapper.Builder { + protected TimeUnit timeUnit = Defaults.TIME_UNIT; + protected String nullValue = Defaults.NULL_VALUE; protected FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER; @@ -74,6 +79,11 @@ public Builder(String name) { builder = this; } + public Builder timeUnit(TimeUnit timeUnit) { + this.timeUnit = timeUnit; + return this; + } + public Builder nullValue(String nullValue) { this.nullValue = nullValue; return this; @@ -86,7 +96,7 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) { @Override public DateFieldMapper build(BuilderContext context) { DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter, - precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue); + precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions, nullValue, timeUnit); fieldMapper.includeInAll(includeInAll); return fieldMapper; } @@ -103,6 +113,8 @@ public static class TypeParser implements Mapper.TypeParser { builder.nullValue(propNode.toString()); } else if (propName.equals("format")) { builder.dateTimeFormatter(parseDateTimeFormatter(propName, propNode)); + } else if (propName.equals("numeric_precision")) { + builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase())); } } return builder; @@ -113,15 +125,18 @@ public static class TypeParser implements Mapper.TypeParser { private String nullValue; + private TimeUnit timeUnit; + protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor, Field.Index index, Field.Store store, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, - String nullValue) { + String nullValue, TimeUnit timeUnit) { super(names, precisionStep, fuzzyFactor, index, store, boost, omitNorms, omitTermFreqAndPositions, new NamedAnalyzer("_date/" + precisionStep, new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())), new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser()))); this.dateTimeFormatter = dateTimeFormatter; this.nullValue = nullValue; + this.timeUnit = timeUnit; } @Override protected double parseFuzzyFactor(String fuzzyFactor) { @@ -265,7 +280,7 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter } if (value != null) { - LongFieldMapper.CustomLongNumericField field = new LongFieldMapper.CustomLongNumericField(this, value); + LongFieldMapper.CustomLongNumericField field = new LongFieldMapper.CustomLongNumericField(this, timeUnit.toMillis(value)); field.setBoost(boost); return field; } @@ -298,6 +313,7 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter } if (!mergeContext.mergeFlags().simulate()) { this.nullValue = ((DateFieldMapper) mergeWith).nullValue; + this.timeUnit = ((DateFieldMapper) mergeWith).timeUnit; } } @@ -331,6 +347,9 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter if (includeInAll != null) { builder.field("include_in_all", includeInAll); } + if (timeUnit != Defaults.TIME_UNIT) { + builder.field("numeric_precision", timeUnit); + } } private long parseStringValue(String value) { From 9bc9e3f0bd4e9b1676da8397246ccd08202bcaab Mon Sep 17 00:00:00 2001 From: Njal Karevoll Date: Tue, 20 Sep 2011 21:09:37 +0200 Subject: [PATCH 45/96] use "numeric_resolution" instead of "numeric_precision" as the field name --- .../org/elasticsearch/index/mapper/core/DateFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 6d8cc1af63faf..334467261914b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -113,7 +113,7 @@ public static class TypeParser implements Mapper.TypeParser { builder.nullValue(propNode.toString()); } else if (propName.equals("format")) { builder.dateTimeFormatter(parseDateTimeFormatter(propName, propNode)); - } else if (propName.equals("numeric_precision")) { + } else if (propName.equals("numeric_resolution")) { builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase())); } } @@ -348,7 +348,7 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter builder.field("include_in_all", includeInAll); } if (timeUnit != Defaults.TIME_UNIT) { - builder.field("numeric_precision", timeUnit); + builder.field("numeric_resolution", timeUnit); } } From 6d975cd6b7862216a288c7785daa5104693b1463 Mon Sep 17 00:00:00 2001 From: Njal Karevoll Date: Wed, 21 Sep 2011 16:40:41 +0200 Subject: [PATCH 46/96] also use the time unit configured by numeric_resolution if fallbacking to parsing a timestamp number --- .../org/elasticsearch/index/mapper/core/DateFieldMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index 334467261914b..e52f84ccb6159 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -357,7 +357,8 @@ private long parseStringValue(String value) { return dateTimeFormatter.parser().parseMillis(value); } catch (RuntimeException e) { try { - return Long.parseLong(value); + long time = Long.parseLong(value); + return timeUnit.toMillis(time); } catch (NumberFormatException e1) { throw new MapperParsingException("failed to parse date field, tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e); } From 7adf2fd497e68ce254e981a26895db58b0fc0c8d Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 00:30:05 +0300 Subject: [PATCH 47/96] make last commit compile... --- .../elasticsearch/index/mapper/core/DateFieldMapper.java | 9 ++++----- .../index/mapper/internal/TimestampFieldMapper.java | 6 ++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java index e52f84ccb6159..85e9df8983d6e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java @@ -121,11 +121,11 @@ public static class TypeParser implements Mapper.TypeParser { } } - private final FormatDateTimeFormatter dateTimeFormatter; + protected final FormatDateTimeFormatter dateTimeFormatter; private String nullValue; - private TimeUnit timeUnit; + protected final TimeUnit timeUnit; protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor, Field.Index index, Field.Store store, @@ -313,7 +313,6 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter } if (!mergeContext.mergeFlags().simulate()) { this.nullValue = ((DateFieldMapper) mergeWith).nullValue; - this.timeUnit = ((DateFieldMapper) mergeWith).timeUnit; } } @@ -348,11 +347,11 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter builder.field("include_in_all", includeInAll); } if (timeUnit != Defaults.TIME_UNIT) { - builder.field("numeric_resolution", timeUnit); + builder.field("numeric_resolution", timeUnit.name().toLowerCase()); } } - private long parseStringValue(String value) { + protected long parseStringValue(String value) { try { return dateTimeFormatter.parser().parseMillis(value); } catch (RuntimeException e) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index bb66f57caef48..dfd5301f25f7c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.support.XContentMapValues.*; import static org.elasticsearch.index.mapper.MapperBuilders.*; @@ -117,8 +118,6 @@ public static class TypeParser implements Mapper.TypeParser { private final String path; - private final FormatDateTimeFormatter dateTimeFormatter; - public TimestampFieldMapper() { this(Defaults.STORE, Defaults.INDEX, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER); } @@ -126,10 +125,9 @@ public TimestampFieldMapper() { protected TimestampFieldMapper(Field.Store store, Field.Index index, boolean enabled, String path, FormatDateTimeFormatter dateTimeFormatter) { super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter, Defaults.PRECISION_STEP, Defaults.FUZZY_FACTOR, index, store, Defaults.BOOST, Defaults.OMIT_NORMS, - Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE); + Defaults.OMIT_TERM_FREQ_AND_POSITIONS, Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/); this.enabled = enabled; this.path = path; - this.dateTimeFormatter = dateTimeFormatter; } public boolean enabled() { From 55d112b0436ce1f02a54ad24fa159b2ae999a576 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 01:00:51 +0300 Subject: [PATCH 48/96] Even shard count distribution counts relocations as two, closes #1354. --- .../cluster/routing/RoutingNodes.java | 19 -------- .../allocator/EvenShardsCountAllocator.java | 47 ++++++++++++++----- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index 6ee83ab92d396..fdedd008a4d80 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -26,9 +26,6 @@ import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; import org.elasticsearch.common.util.concurrent.NotThreadSafe; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -233,22 +230,6 @@ public List shardsWithState(String index, ShardRoutingState return shards; } - public List sortedNodesLeastToHigh() { - return nodesToShardsSorted(new Comparator() { - @Override public int compare(RoutingNode o1, RoutingNode o2) { - return o1.shards().size() - o2.shards().size(); - } - }); - } - - public List nodesToShardsSorted(Comparator comparator) { - List nodes = new ArrayList(nodesToShards.values()); - if (comparator != null) { - Collections.sort(nodes, comparator); - } - return nodes; - } - public String prettyPrint() { StringBuilder sb = new StringBuilder("routing_nodes:\n"); for (RoutingNode routingNode : this) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/EvenShardsCountAllocator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/EvenShardsCountAllocator.java index b9c4c7cae5243..e5bd4fb7829e2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/EvenShardsCountAllocator.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/EvenShardsCountAllocator.java @@ -22,13 +22,17 @@ import org.elasticsearch.cluster.routing.MutableShardRouting; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.RoutingNodes; +import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.allocation.FailedRerouteAllocation; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.StartedRerouteAllocation; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; +import java.util.Arrays; +import java.util.Comparator; import java.util.Iterator; import java.util.List; @@ -53,7 +57,7 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard RoutingNodes routingNodes = allocation.routingNodes(); - List nodes = routingNodes.sortedNodesLeastToHigh(); + RoutingNode[] nodes = sortedNodesLeastToHigh(allocation); Iterator unassignedIterator = routingNodes.unassigned().iterator(); int lastNode = 0; @@ -61,10 +65,10 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard while (unassignedIterator.hasNext()) { MutableShardRouting shard = unassignedIterator.next(); // do the allocation, finding the least "busy" node - for (int i = 0; i < nodes.size(); i++) { - RoutingNode node = nodes.get(lastNode); + for (int i = 0; i < nodes.length; i++) { + RoutingNode node = nodes[lastNode]; lastNode++; - if (lastNode == nodes.size()) { + if (lastNode == nodes.length) { lastNode = 0; } @@ -86,7 +90,7 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard for (Iterator it = routingNodes.unassigned().iterator(); it.hasNext(); ) { MutableShardRouting shard = it.next(); // go over the nodes and try and allocate the remaining ones - for (RoutingNode routingNode : routingNodes.sortedNodesLeastToHigh()) { + for (RoutingNode routingNode : sortedNodesLeastToHigh(allocation)) { if (allocation.deciders().canAllocate(shard, routingNode, allocation).allocate()) { changed = true; routingNode.add(shard); @@ -100,18 +104,18 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard @Override public boolean rebalance(RoutingAllocation allocation) { boolean changed = false; - List sortedNodesLeastToHigh = allocation.routingNodes().sortedNodesLeastToHigh(); - if (sortedNodesLeastToHigh.isEmpty()) { + RoutingNode[] sortedNodesLeastToHigh = sortedNodesLeastToHigh(allocation); + if (sortedNodesLeastToHigh.length == 0) { return false; } int lowIndex = 0; - int highIndex = sortedNodesLeastToHigh.size() - 1; + int highIndex = sortedNodesLeastToHigh.length - 1; boolean relocationPerformed; do { relocationPerformed = false; while (lowIndex != highIndex) { - RoutingNode lowRoutingNode = sortedNodesLeastToHigh.get(lowIndex); - RoutingNode highRoutingNode = sortedNodesLeastToHigh.get(highIndex); + RoutingNode lowRoutingNode = sortedNodesLeastToHigh[lowIndex]; + RoutingNode highRoutingNode = sortedNodesLeastToHigh[highIndex]; int averageNumOfShards = allocation.routingNodes().requiredAverageNumberOfShardsPerNode(); // only active shards can be removed so must count only active ones. @@ -156,8 +160,8 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard @Override public boolean move(MutableShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { assert shardRouting.started(); boolean changed = false; - List sortedNodesLeastToHigh = allocation.routingNodes().sortedNodesLeastToHigh(); - if (sortedNodesLeastToHigh.isEmpty()) { + RoutingNode[] sortedNodesLeastToHigh = sortedNodesLeastToHigh(allocation); + if (sortedNodesLeastToHigh.length == 0) { return false; } @@ -179,4 +183,23 @@ public class EvenShardsCountAllocator extends AbstractComponent implements Shard return changed; } + + private RoutingNode[] sortedNodesLeastToHigh(RoutingAllocation allocation) { + // create count per node id, taking into account relocations + final TObjectIntHashMap nodeCounts = new TObjectIntHashMap(); + for (RoutingNode node : allocation.routingNodes()) { + for (int i = 0; i < node.shards().size(); i++) { + ShardRouting shardRouting = node.shards().get(i); + String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId(); + nodeCounts.adjustOrPutValue(nodeId, 1, 1); + } + } + RoutingNode[] nodes = allocation.routingNodes().nodesToShards().values().toArray(new RoutingNode[allocation.routingNodes().nodesToShards().values().size()]); + Arrays.sort(nodes, new Comparator() { + @Override public int compare(RoutingNode o1, RoutingNode o2) { + return nodeCounts.get(o1.nodeId()) - nodeCounts.get(o2.nodeId()); + } + }); + return nodes; + } } From f74fa7511bdf87bae3a340820151f31f3ce07792 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 13:53:40 +0300 Subject: [PATCH 49/96] better failure when failing to find mapping for the key field in data histogram --- .../facet/datehistogram/DateHistogramFacetProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java index 9acdf7774842a..0df6bb8387ab7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetProcessor.java @@ -153,6 +153,9 @@ public class DateHistogramFacetProcessor extends AbstractComponent implements Fa } FieldMapper mapper = context.mapperService().smartNameFieldMapper(keyField); + if (mapper == null) { + throw new FacetPhaseExecutionException(facetName, "(key) field [" + keyField + "] not found"); + } if (mapper.fieldDataType() != FieldDataType.DefaultTypes.LONG) { throw new FacetPhaseExecutionException(facetName, "(key) field [" + keyField + "] is not of type date"); } From 7ab128bbba7bb11248a93816a2a9b3f59a4a2e83 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 16:02:18 +0300 Subject: [PATCH 50/96] Put mapping on a single node with new mapping will not wait for the mapping to be applied, closes #1355. --- .../cluster/metadata/MetaDataMappingService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java index af79baf7514ab..f7b8d7932f8d8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java @@ -313,6 +313,10 @@ public void putMapping(final PutRequest request, final Listener listener) { } else { CompressedString newSource = newMapper.mappingSource(); mappings.put(index, new MappingMetaData(newMapper)); + // we also add it to the registered parsed mapping, since that's what we do when we merge + // and, we won't wait for it to be created on this master node + IndexService indexService = indicesService.indexService(index); + indexService.mapperService().add(newMapper.type(), newMapper.mappingSource().string()); if (logger.isDebugEnabled()) { logger.debug("[{}] create_mapping [{}] with source [{}]", index, newMapper.type(), newSource); } else if (logger.isInfoEnabled()) { From 25fe56c462a4fa77249de49bb3a5100904256905 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 16:11:22 +0300 Subject: [PATCH 51/96] more javadoc --- .../src/main/java/org/elasticsearch/wares/NodeServlet.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/transport/wares/src/main/java/org/elasticsearch/wares/NodeServlet.java b/plugins/transport/wares/src/main/java/org/elasticsearch/wares/NodeServlet.java index 53f8e1cde3702..e56648728efeb 100644 --- a/plugins/transport/wares/src/main/java/org/elasticsearch/wares/NodeServlet.java +++ b/plugins/transport/wares/src/main/java/org/elasticsearch/wares/NodeServlet.java @@ -45,6 +45,8 @@ * *

The node is registered as a servlet context attribute under elasticsearchNode so its easily * accessible from other web resources if needed. + * + *

The servlet can be registered under a prefix URI, and it will automatically adjust to handle it. */ public class NodeServlet extends HttpServlet { From c1ca21f4d5b683ad03180fa7e5b8673c51221aee Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Thu, 22 Sep 2011 18:04:59 +0300 Subject: [PATCH 52/96] more internal refactoring in directory providers --- .../index/store/DirectoryService.java | 2 +- .../org/elasticsearch/index/store/Store.java | 177 ++++++++++++------ .../index/store/StoreFileMetaData.java | 12 ++ .../store/fs/MmapFsDirectoryService.java | 4 +- .../index/store/fs/NioFsDirectoryService.java | 4 +- .../store/fs/SimpleFsDirectoryService.java | 4 +- .../memory/ByteBufferDirectoryService.java | 4 +- .../index/store/ram/RamDirectoryService.java | 4 +- 8 files changed, 143 insertions(+), 68 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java index f85ed364168f7..5687a34b1a9b0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/DirectoryService.java @@ -27,7 +27,7 @@ */ public interface DirectoryService { - Directory build() throws IOException; + Directory[] build() throws IOException; void renameFile(Directory dir, String from, String to) throws IOException; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java index 04fbafa37b291..9bbd67e61a9a2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java @@ -20,14 +20,17 @@ package org.elasticsearch.index.store; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; import org.elasticsearch.common.Strings; import org.elasticsearch.common.Unicode; +import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.collect.Maps; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.Directories; import org.elasticsearch.common.settings.Settings; @@ -39,6 +42,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -131,7 +135,9 @@ public void deleteContent() throws IOException { public void fullDelete() throws IOException { deleteContent(); - directoryService.fullDelete(directory.delegate()); + for (Directory delegate : directory.delegates()) { + directoryService.fullDelete(delegate); + } } public StoreStats stats() throws IOException { @@ -143,10 +149,13 @@ public ByteSizeValue estimateSize() throws IOException { } public void renameFile(String from, String to) throws IOException { - directoryService.renameFile(directory.delegate(), from, to); synchronized (mutex) { StoreFileMetaData fromMetaData = filesMetadata.get(from); // we should always find this one - StoreFileMetaData toMetaData = new StoreFileMetaData(to, fromMetaData.length(), fromMetaData.lastModified(), fromMetaData.checksum()); + if (fromMetaData == null) { + throw new FileNotFoundException(from); + } + directoryService.renameFile(fromMetaData.directory(), from, to); + StoreFileMetaData toMetaData = new StoreFileMetaData(to, fromMetaData.length(), fromMetaData.lastModified(), fromMetaData.checksum(), fromMetaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).remove(from).put(to, toMetaData).immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); } @@ -227,7 +236,7 @@ public void writeChecksum(String name, String checksum) throws IOException { // update the metadata to include the checksum and write a new checksums file synchronized (mutex) { StoreFileMetaData metaData = filesMetadata.get(name); - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), checksum); + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), checksum, metaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); writeChecksums(); } @@ -238,7 +247,7 @@ public void writeChecksums(Map checksums) throws IOException { synchronized (mutex) { for (Map.Entry entry : checksums.entrySet()) { StoreFileMetaData metaData = filesMetadata.get(entry.getKey()); - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), entry.getValue()); + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.lastModified(), entry.getValue(), metaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(entry.getKey(), metaData).immutableMap(); } writeChecksums(); @@ -250,44 +259,46 @@ public void writeChecksums(Map checksums) throws IOException { */ protected class StoreDirectory extends Directory implements ForceSyncDirectory { - private final Directory delegate; + private final Directory[] delegates; - StoreDirectory(Directory delegate) throws IOException { - this.delegate = delegate; + StoreDirectory(Directory[] delegates) throws IOException { + this.delegates = delegates; synchronized (mutex) { - Map checksums = readChecksums(delegate); MapBuilder builder = MapBuilder.newMapBuilder(); - for (String file : delegate.listAll()) { - // BACKWARD CKS SUPPORT - if (file.endsWith(".cks")) { // ignore checksum files here - continue; - } - String checksum = checksums.get(file); - - // BACKWARD CKS SUPPORT - if (checksum == null) { - if (delegate.fileExists(file + ".cks")) { - IndexInput indexInput = delegate.openInput(file + ".cks"); - try { - if (indexInput.length() > 0) { - byte[] checksumBytes = new byte[(int) indexInput.length()]; - indexInput.readBytes(checksumBytes, 0, checksumBytes.length, false); - checksum = Unicode.fromBytes(checksumBytes); + Map checksums = readChecksums(delegates[0]); + for (Directory delegate : delegates) { + for (String file : delegate.listAll()) { + // BACKWARD CKS SUPPORT + if (file.endsWith(".cks")) { // ignore checksum files here + continue; + } + String checksum = checksums.get(file); + + // BACKWARD CKS SUPPORT + if (checksum == null) { + if (delegate.fileExists(file + ".cks")) { + IndexInput indexInput = delegate.openInput(file + ".cks"); + try { + if (indexInput.length() > 0) { + byte[] checksumBytes = new byte[(int) indexInput.length()]; + indexInput.readBytes(checksumBytes, 0, checksumBytes.length, false); + checksum = Unicode.fromBytes(checksumBytes); + } + } finally { + indexInput.close(); } - } finally { - indexInput.close(); } } + builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), checksum, delegate)); } - builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), checksum)); } filesMetadata = builder.immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); } } - public Directory delegate() { - return delegate; + public Directory[] delegates() { + return delegates; } @Override public String[] listAll() throws IOException { @@ -307,15 +318,15 @@ public Directory delegate() { if (metaData.lastModified() != -1) { return metaData.lastModified(); } - return delegate.fileModified(name); + return metaData.directory().fileModified(name); } @Override public void touchFile(String name) throws IOException { - delegate.touchFile(name); synchronized (mutex) { StoreFileMetaData metaData = filesMetadata.get(name); if (metaData != null) { - metaData = new StoreFileMetaData(metaData.name(), metaData.length(), delegate.fileModified(name), metaData.checksum()); + metaData.directory().touchFile(name); + metaData = new StoreFileMetaData(metaData.name(), metaData.length(), metaData.directory().fileModified(name), metaData.checksum(), metaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); } } @@ -323,9 +334,9 @@ public Directory delegate() { public void deleteFileChecksum(String name) throws IOException { try { - delegate.deleteFile(name); + delegates[0].deleteFile(name); } catch (IOException e) { - if (delegate.fileExists(name)) { + if (delegates[0].fileExists(name)) { throw e; } } @@ -340,11 +351,14 @@ public void deleteFileChecksum(String name) throws IOException { if (isChecksum(name)) { return; } - try { - delegate.deleteFile(name); - } catch (IOException e) { - if (delegate.fileExists(name)) { - throw e; + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData != null) { + try { + metaData.directory().deleteFile(name); + } catch (IOException e) { + if (metaData.directory().fileExists(name)) { + throw e; + } } } synchronized (mutex) { @@ -362,7 +376,7 @@ public void deleteFileChecksum(String name) throws IOException { if (metaData.length() != -1) { return metaData.length(); } - return delegate.fileLength(name); + return metaData.directory().fileLength(name); } @Override public IndexOutput createOutput(String name) throws IOException { @@ -370,21 +384,48 @@ public void deleteFileChecksum(String name) throws IOException { } public IndexOutput createOutput(String name, boolean computeChecksum) throws IOException { - IndexOutput out = delegate.createOutput(name); + Directory directory = null; + if (isChecksum(name)) { + directory = delegates[0]; + } else { + if (delegates.length == 1) { + directory = delegates[0]; + } else { + long size = Long.MAX_VALUE; + for (Directory delegate : delegates) { + if (delegate instanceof FSDirectory) { + long currentSize = ((FSDirectory) delegate).getDirectory().getFreeSpace(); + if (currentSize < size) { + size = currentSize; + directory = delegate; + } + } else { + directory = delegate; // really, make sense to have multiple directories for FS + } + } + } + } + IndexOutput out = directory.createOutput(name); synchronized (mutex) { - StoreFileMetaData metaData = new StoreFileMetaData(name, -1, -1, null); + StoreFileMetaData metaData = new StoreFileMetaData(name, -1, -1, null, directory); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); + return new StoreIndexOutput(metaData, out, name, computeChecksum); } - return new StoreIndexOutput(out, name, computeChecksum); } @Override public IndexInput openInput(String name) throws IOException { - return delegate.openInput(name); + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData == null) { + throw new FileNotFoundException(name); + } + return metaData.directory().openInput(name); } @Override public void close() throws IOException { - delegate.close(); + for (Directory delegate : delegates) { + delegate.close(); + } synchronized (mutex) { filesMetadata = ImmutableMap.of(); files = Strings.EMPTY_ARRAY; @@ -392,32 +433,51 @@ public IndexOutput createOutput(String name, boolean computeChecksum) throws IOE } @Override public Lock makeLock(String name) { - return delegate.makeLock(name); + return delegates[0].makeLock(name); } @Override public IndexInput openInput(String name, int bufferSize) throws IOException { - return delegate.openInput(name, bufferSize); + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData == null) { + throw new FileNotFoundException(name); + } + return metaData.directory().openInput(name, bufferSize); } @Override public void clearLock(String name) throws IOException { - delegate.clearLock(name); + delegates[0].clearLock(name); } @Override public void setLockFactory(LockFactory lockFactory) throws IOException { - delegate.setLockFactory(lockFactory); + delegates[0].setLockFactory(lockFactory); } @Override public LockFactory getLockFactory() { - return delegate.getLockFactory(); + return delegates[0].getLockFactory(); } @Override public String getLockID() { - return delegate.getLockID(); + return delegates[0].getLockID(); } @Override public void sync(Collection names) throws IOException { if (sync) { - delegate.sync(names); + Map> map = Maps.newHashMap(); + for (String name : names) { + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData == null) { + throw new FileNotFoundException(name); + } + Collection dirNames = map.get(metaData.directory()); + if (dirNames == null) { + dirNames = new ArrayList(); + map.put(metaData.directory(), dirNames); + } + dirNames.add(name); + } + for (Map.Entry> entry : map.entrySet()) { + entry.getKey().sync(entry.getValue()); + } } for (String name : names) { // write the checksums file when we sync on the segments file (committed) @@ -430,7 +490,7 @@ public IndexOutput createOutput(String name, boolean computeChecksum) throws IOE @Override public void sync(String name) throws IOException { if (sync) { - delegate.sync(name); + sync(ImmutableList.of(name)); } // write the checksums file when we sync on the segments file (committed) if (!name.equals("segments.gen") && name.startsWith("segments")) { @@ -439,19 +499,22 @@ public IndexOutput createOutput(String name, boolean computeChecksum) throws IOE } @Override public void forceSync(String name) throws IOException { - delegate.sync(name); + sync(ImmutableList.of(name)); } } class StoreIndexOutput extends IndexOutput { + private final StoreFileMetaData metaData; + private final IndexOutput delegate; private final String name; private final Checksum digest; - StoreIndexOutput(IndexOutput delegate, String name, boolean computeChecksum) { + StoreIndexOutput(StoreFileMetaData metaData, IndexOutput delegate, String name, boolean computeChecksum) { + this.metaData = metaData; this.delegate = delegate; this.name = name; if (computeChecksum) { @@ -480,7 +543,7 @@ class StoreIndexOutput extends IndexOutput { checksum = Long.toString(digest.getValue(), Character.MAX_RADIX); } synchronized (mutex) { - StoreFileMetaData md = new StoreFileMetaData(name, directory.delegate().fileLength(name), directory.delegate().fileModified(name), checksum); + StoreFileMetaData md = new StoreFileMetaData(name, metaData.directory().fileLength(name), metaData.directory().fileModified(name), checksum, metaData.directory()); filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, md).immutableMap(); files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java index c1b7815b5e9ae..8605a2d3c6f9a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/StoreFileMetaData.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store; +import org.apache.lucene.store.Directory; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -39,14 +40,25 @@ public class StoreFileMetaData implements Streamable { private String checksum; + private transient Directory directory; + StoreFileMetaData() { } public StoreFileMetaData(String name, long length, long lastModified, String checksum) { + this(name, length, lastModified, checksum, null); + } + + public StoreFileMetaData(String name, long length, long lastModified, String checksum, @Nullable Directory directory) { this.name = name; this.lastModified = lastModified; this.length = length; this.checksum = checksum; + this.directory = directory; + } + + public Directory directory() { + return this.directory; } public String name() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java index 336e82d0ca05a..c54066fc16a06 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java @@ -39,9 +39,9 @@ public class MmapFsDirectoryService extends FsDirectoryService { super(shardId, indexSettings, indexStore); } - @Override public Directory build() throws IOException { + @Override public Directory[] build() throws IOException { File location = indexStore.shardIndexLocation(shardId); FileSystemUtils.mkdirs(location); - return new MMapDirectory(location, buildLockFactory()); + return new Directory[]{new MMapDirectory(location, buildLockFactory())}; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java index 5d148347c75fc..e020b9a349e8a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java @@ -39,9 +39,9 @@ public class NioFsDirectoryService extends FsDirectoryService { super(shardId, indexSettings, indexStore); } - @Override public Directory build() throws IOException { + @Override public Directory[] build() throws IOException { File location = indexStore.shardIndexLocation(shardId); FileSystemUtils.mkdirs(location); - return new NIOFSDirectory(location, buildLockFactory()); + return new Directory[]{new NIOFSDirectory(location, buildLockFactory())}; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java index 7fec5e31e2e4f..dee62bf263714 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java @@ -39,9 +39,9 @@ public class SimpleFsDirectoryService extends FsDirectoryService { super(shardId, indexSettings, indexStore); } - @Override public Directory build() throws IOException { + @Override public Directory[] build() throws IOException { File location = indexStore.shardIndexLocation(shardId); FileSystemUtils.mkdirs(location); - return new SimpleFSDirectory(location, buildLockFactory()); + return new Directory[]{new SimpleFSDirectory(location, buildLockFactory())}; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java index 964527a5cc075..3a99a0b477b47 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferDirectoryService.java @@ -46,8 +46,8 @@ public class ByteBufferDirectoryService extends AbstractIndexShardComponent impl this.byteBufferCache = byteBufferCache; } - @Override public Directory build() { - return new CustomByteBufferDirectory(byteBufferCache); + @Override public Directory[] build() { + return new Directory[]{new CustomByteBufferDirectory(byteBufferCache)}; } @Override public void renameFile(Directory dir, String from, String to) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java index 5b8dea1d99e2e..c598cb7c3522e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamDirectoryService.java @@ -40,8 +40,8 @@ public class RamDirectoryService extends AbstractIndexShardComponent implements super(shardId, indexSettings); } - @Override public Directory build() { - return new CustomRAMDirectory(); + @Override public Directory[] build() { + return new Directory[]{new CustomRAMDirectory()}; } @Override public void renameFile(Directory dir, String from, String to) throws IOException { From 8d7aaa704a644569359e6a1cddc9e2c10b34da79 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 23 Sep 2011 00:35:59 +0300 Subject: [PATCH 53/96] Allow to stripe the data location over multiple locations, closes #1356. --- .../common/io/FileSystemUtils.java | 8 ++ .../common/settings/ImmutableSettings.java | 1 + .../org/elasticsearch/env/Environment.java | 26 ++-- .../elasticsearch/env/NodeEnvironment.java | 134 ++++++++++++------ .../elasticsearch/gateway/fs/FsGateway.java | 2 +- .../gateway/local/LocalGateway.java | 5 +- .../gateway/local/LocalIndexShardGateway.java | 27 ++-- .../index/service/InternalIndexService.java | 2 +- .../org/elasticsearch/index/store/Store.java | 27 +++- .../index/store/fs/FsIndexStore.java | 57 +++++--- .../store/fs/MmapFsDirectoryService.java | 10 +- .../index/store/fs/NioFsDirectoryService.java | 10 +- .../store/fs/SimpleFsDirectoryService.java | 10 +- .../index/translog/fs/FsTranslog.java | 69 ++++++--- .../indices/InternalIndicesService.java | 2 +- .../indices/store/IndicesStore.java | 27 ++-- .../TransportNodesListShardStoreMetaData.java | 58 ++++---- .../internal/InternalSettingsPerparer.java | 8 +- .../fs/AbstractSimpleIndexGatewayTests.java | 4 +- .../indices/store/IndicesStoreTests.java | 5 +- .../fullrestart/FullRestartStressTest.java | 5 +- .../RollingRestartStressTest.java | 5 +- 22 files changed, 325 insertions(+), 177 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index a5efd0e911386..d8cad05f09d2f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -120,6 +120,14 @@ public static boolean hasExtensions(File root, String... extensions) { return false; } + public static boolean deleteRecursively(File[] roots) { + boolean deleted = true; + for (File root : roots) { + deleted &= deleteRecursively(root); + } + return deleted; + } + public static boolean deleteRecursively(File root) { return deleteRecursively(root, true); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index d259c5dad61a7..e917cf8437b63 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -492,6 +492,7 @@ public Builder put(String setting, long value, ByteSizeUnit sizeUnit) { * @return The builder */ public Builder putArray(String setting, String... values) { + remove(setting); int counter = 0; while (true) { String value = map.remove(setting + '.' + (counter++)); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/env/Environment.java b/modules/elasticsearch/src/main/java/org/elasticsearch/env/Environment.java index 953d3de29be09..ae1826bc12cb8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/env/Environment.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/env/Environment.java @@ -46,9 +46,9 @@ public class Environment { private final File workWithClusterFile; - private final File dataFile; + private final File[] dataFiles; - private final File dataWithClusterFile; + private final File[] dataWithClusterFiles; private final File configFile; @@ -86,12 +86,18 @@ public Environment(Settings settings) { } workWithClusterFile = new File(workFile, ClusterName.clusterNameFromSettings(settings).value()); - if (settings.get("path.data") != null) { - dataFile = new File(cleanPath(settings.get("path.data"))); + String[] dataPaths = settings.getAsArray("path.data"); + if (dataPaths.length > 0) { + dataFiles = new File[dataPaths.length]; + dataWithClusterFiles = new File[dataPaths.length]; + for (int i = 0; i < dataPaths.length; i++) { + dataFiles[i] = new File(dataPaths[i]); + dataWithClusterFiles[i] = new File(dataFiles[i], ClusterName.clusterNameFromSettings(settings).value()); + } } else { - dataFile = new File(homeFile, "data"); + dataFiles = new File[]{new File(homeFile, "data")}; + dataWithClusterFiles = new File[]{new File(new File(homeFile, "data"), ClusterName.clusterNameFromSettings(settings).value())}; } - dataWithClusterFile = new File(dataFile, ClusterName.clusterNameFromSettings(settings).value()); if (settings.get("path.logs") != null) { logsFile = new File(cleanPath(settings.get("path.logs"))); @@ -124,15 +130,15 @@ public File workWithClusterFile() { /** * The data location. */ - public File dataFile() { - return dataFile; + public File[] dataFiles() { + return dataFiles; } /** * The data location with the cluster name as a sub directory. */ - public File dataWithClusterFile() { - return dataWithClusterFile; + public File[] dataWithClusterFiles() { + return dataWithClusterFiles; } /** diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java index f7bf85f32c59e..3a4c273f67385 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -38,9 +38,10 @@ */ public class NodeEnvironment extends AbstractComponent { - private final File nodeFile; + private final File[] nodeFiles; + private final File[] nodeIndicesLocations; - private final Lock lock; + private final Lock[] locks; private final int localNodeId; @@ -48,46 +49,83 @@ public class NodeEnvironment extends AbstractComponent { super(settings); if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) { - nodeFile = null; - lock = null; + nodeFiles = null; + nodeIndicesLocations = null; + locks = null; localNodeId = -1; return; } - Lock lock = null; - File dir = null; + File[] nodesFiles = new File[environment.dataWithClusterFiles().length]; + Lock[] locks = new Lock[environment.dataWithClusterFiles().length]; int localNodeId = -1; IOException lastException = null; - for (int i = 0; i < 50; i++) { - dir = new File(new File(environment.dataWithClusterFile(), "nodes"), Integer.toString(i)); - if (!dir.exists()) { - FileSystemUtils.mkdirs(dir); - } - logger.trace("obtaining node lock on {} ...", dir.getAbsolutePath()); - try { - NativeFSLockFactory lockFactory = new NativeFSLockFactory(dir); - Lock tmpLock = lockFactory.makeLock("node.lock"); - boolean obtained = tmpLock.obtain(); - if (obtained) { - lock = tmpLock; - localNodeId = i; + for (int possibleLockId = 0; possibleLockId < 50; possibleLockId++) { + for (int dirIndex = 0; dirIndex < environment.dataWithClusterFiles().length; dirIndex++) { + File dir = new File(new File(environment.dataWithClusterFiles()[dirIndex], "nodes"), Integer.toString(possibleLockId)); + if (!dir.exists()) { + FileSystemUtils.mkdirs(dir); + } + logger.trace("obtaining node lock on {} ...", dir.getAbsolutePath()); + try { + NativeFSLockFactory lockFactory = new NativeFSLockFactory(dir); + Lock tmpLock = lockFactory.makeLock("node.lock"); + boolean obtained = tmpLock.obtain(); + if (obtained) { + locks[dirIndex] = tmpLock; + nodesFiles[dirIndex] = dir; + localNodeId = possibleLockId; + } else { + logger.trace("failed to obtain node lock on {}", dir.getAbsolutePath()); + // release all the ones that were obtained up until now + for (int i = 0; i < locks.length; i++) { + if (locks[i] != null) { + try { + locks[i].release(); + } catch (Exception e1) { + // ignore + } + } + locks[i] = null; + } + break; + } + } catch (IOException e) { + logger.trace("failed to obtain node lock on {}", e, dir.getAbsolutePath()); + lastException = e; + // release all the ones that were obtained up until now + for (int i = 0; i < locks.length; i++) { + if (locks[i] != null) { + try { + locks[i].release(); + } catch (Exception e1) { + // ignore + } + } + locks[i] = null; + } break; - } else { - logger.trace("failed to obtain node lock on {}", dir.getAbsolutePath()); } - } catch (IOException e) { - logger.trace("failed to obtain node lock on {}", e, dir.getAbsolutePath()); - lastException = e; + } + if (locks[0] != null) { + // we found a lock, break + break; } } - if (lock == null) { + if (locks[0] == null) { throw new IOException("Failed to obtain node lock", lastException); } + this.localNodeId = localNodeId; - this.lock = lock; - this.nodeFile = dir; + this.locks = locks; + this.nodeFiles = nodesFiles; if (logger.isDebugEnabled()) { - logger.debug("using node location [{}], local_node_id [{}]", dir, localNodeId); + logger.debug("using node location [{}], local_node_id [{}]", nodesFiles, localNodeId); + } + + this.nodeIndicesLocations = new File[nodeFiles.length]; + for (int i = 0; i < nodeFiles.length; i++) { + nodeIndicesLocations[i] = new File(nodeFiles[i], "indices"); } } @@ -96,34 +134,44 @@ public int localNodeId() { } public boolean hasNodeFile() { - return nodeFile != null && lock != null; + return nodeFiles != null && locks != null; } - public File nodeDataLocation() { - if (nodeFile == null || lock == null) { + public File[] nodeDataLocations() { + if (nodeFiles == null || locks == null) { throw new ElasticSearchIllegalStateException("node is not configured to store local location"); } - return nodeFile; + return nodeFiles; } - public File indicesLocation() { - return new File(nodeDataLocation(), "indices"); + public File[] indicesLocations() { + return nodeIndicesLocations; } - public File indexLocation(Index index) { - return new File(indicesLocation(), index.name()); + public File[] indexLocations(Index index) { + File[] indexLocations = new File[nodeFiles.length]; + for (int i = 0; i < nodeFiles.length; i++) { + indexLocations[i] = new File(new File(nodeFiles[i], "indices"), index.name()); + } + return indexLocations; } - public File shardLocation(ShardId shardId) { - return new File(indexLocation(shardId.index()), Integer.toString(shardId.id())); + public File[] shardLocations(ShardId shardId) { + File[] shardLocations = new File[nodeFiles.length]; + for (int i = 0; i < nodeFiles.length; i++) { + shardLocations[i] = new File(new File(new File(nodeFiles[i], "indices"), shardId.index().name()), Integer.toString(shardId.id())); + } + return shardLocations; } public void close() { - if (lock != null) { - try { - lock.release(); - } catch (IOException e) { - // ignore + if (locks != null) { + for (Lock lock : locks) { + try { + lock.release(); + } catch (IOException e) { + // ignore + } } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java index e4ff0e3432c4b..102b0888bceda 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/fs/FsGateway.java @@ -53,7 +53,7 @@ public class FsGateway extends BlobStoreGateway { String location = componentSettings.get("location"); if (location == null) { logger.warn("using local fs location for gateway, should be changed to be a shared location across nodes"); - gatewayFile = new File(environment.dataFile(), "gateway"); + gatewayFile = new File(environment.dataFiles()[0], "gateway"); } else { gatewayFile = new File(location); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index 758d9d77266c0..1a96c94949517 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -180,7 +180,7 @@ public LocalGatewayStartedShards currentStartedShards() { } @Override public void reset() throws Exception { - FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocation()); + FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocations()); } @Override public void clusterChanged(final ClusterChangedEvent event) { @@ -263,7 +263,8 @@ private synchronized void lazyInitialize() { location = null; } else { // create the location where the state will be stored - this.location = new File(nodeEnv.nodeDataLocation(), "_state"); + // TODO: we might want to persist states on all data locations + this.location = new File(nodeEnv.nodeDataLocations()[0], "_state"); FileSystemUtils.mkdirs(this.location); if (clusterService.localNode().masterNode()) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java index f008039b28fb0..6e42255ff6e31 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java @@ -128,19 +128,30 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen // move an existing translog, if exists, to "recovering" state, and start reading from it FsTranslog translog = (FsTranslog) indexShard.translog(); - File recoveringTranslogFile = new File(translog.location(), "translog-" + translogId + ".recovering"); - if (!recoveringTranslogFile.exists()) { - File translogFile = new File(translog.location(), "translog-" + translogId); - if (translogFile.exists()) { - for (int i = 0; i < 3; i++) { - if (translogFile.renameTo(recoveringTranslogFile)) { - break; + String translogName = "translog-" + translogId; + String recoverTranslogName = translogName + ".recovering"; + + + File recoveringTranslogFile = null; + for (File translogLocation : translog.locations()) { + File tmpRecoveringFile = new File(translogLocation, recoverTranslogName); + if (!tmpRecoveringFile.exists()) { + File tmpTranslogFile = new File(translogLocation, translogName); + if (tmpTranslogFile.exists()) { + for (int i = 0; i < 3; i++) { + if (tmpTranslogFile.renameTo(tmpRecoveringFile)) { + recoveringTranslogFile = tmpRecoveringFile; + break; + } } } + } else { + recoveringTranslogFile = tmpRecoveringFile; + break; } } - if (!recoveringTranslogFile.exists()) { + if (recoveringTranslogFile == null || !recoveringTranslogFile.exists()) { // no translog to recovery from, start and bail // no translog files, bail indexShard.start("post recovery from gateway, no translog"); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/service/InternalIndexService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/service/InternalIndexService.java index ce3882a50209c..12cf19c415cdb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/service/InternalIndexService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/service/InternalIndexService.java @@ -436,7 +436,7 @@ private void deleteShard(int shardId, boolean delete, boolean snapshotGateway, b // delete the shard location if needed if (delete || indexGateway.type().equals(NoneGateway.TYPE)) { - FileSystemUtils.deleteRecursively(nodeEnv.shardLocation(sId)); + FileSystemUtils.deleteRecursively(nodeEnv.shardLocations(sId)); } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java index 9bbd67e61a9a2..d890a03b36516 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java @@ -35,11 +35,13 @@ import org.elasticsearch.common.lucene.Directories; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.support.ForceSyncDirectory; +import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -161,7 +163,22 @@ public void renameFile(String from, String to) throws IOException { } } - public static Map readChecksums(Directory dir) throws IOException { + public static Map readChecksums(File[] locations) throws IOException { + for (File location : locations) { + FSDirectory directory = FSDirectory.open(location); + try { + Map checksums = readChecksums(directory, null); + if (checksums != null) { + return checksums; + } + } finally { + directory.close(); + } + } + return null; + } + + static Map readChecksums(Directory dir, Map defaultValue) throws IOException { long lastFound = -1; for (String name : dir.listAll()) { if (!isChecksum(name)) { @@ -173,7 +190,7 @@ public static Map readChecksums(Directory dir) throws IOExceptio } } if (lastFound == -1) { - return ImmutableMap.of(); + return defaultValue; } IndexInput indexInput = dir.openInput(CHECKSUMS_PREFIX + lastFound); try { @@ -181,7 +198,7 @@ public static Map readChecksums(Directory dir) throws IOExceptio return indexInput.readStringStringMap(); } catch (Exception e) { // failed to load checksums, ignore and return an empty map - return new HashMap(); + return defaultValue; } finally { indexInput.close(); } @@ -265,7 +282,7 @@ protected class StoreDirectory extends Directory implements ForceSyncDirectory { this.delegates = delegates; synchronized (mutex) { MapBuilder builder = MapBuilder.newMapBuilder(); - Map checksums = readChecksums(delegates[0]); + Map checksums = readChecksums(delegates[0], new HashMap()); for (Directory delegate : delegates) { for (String file : delegate.listAll()) { // BACKWARD CKS SUPPORT @@ -398,6 +415,8 @@ public IndexOutput createOutput(String name, boolean computeChecksum) throws IOE if (currentSize < size) { size = currentSize; directory = delegate; + } else if (currentSize == size && ThreadLocalRandom.current().nextBoolean()) { + directory = delegate; } } else { directory = delegate; // really, make sense to have multiple directories for FS diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java index 46135e702827f..c184135572a4a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java @@ -40,15 +40,15 @@ public abstract class FsIndexStore extends AbstractIndexStore { private final NodeEnvironment nodeEnv; - private final File location; + private final File[] locations; public FsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, NodeEnvironment nodeEnv) { super(index, indexSettings, indexService); this.nodeEnv = nodeEnv; if (nodeEnv.hasNodeFile()) { - this.location = nodeEnv.indexLocation(index); + this.locations = nodeEnv.indexLocations(index); } else { - this.location = null; + this.locations = null; } } @@ -57,58 +57,73 @@ public FsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexSer } @Override public ByteSizeValue backingStoreTotalSpace() { - if (location == null) { + if (locations == null) { return new ByteSizeValue(0); } - long totalSpace = location.getTotalSpace(); - if (totalSpace == 0) { - totalSpace = 0; + long totalSpace = 0; + for (File location : locations) { + totalSpace += location.getTotalSpace(); } return new ByteSizeValue(totalSpace); } @Override public ByteSizeValue backingStoreFreeSpace() { - if (location == null) { + if (locations == null) { return new ByteSizeValue(0); } - long usableSpace = location.getUsableSpace(); - if (usableSpace == 0) { - usableSpace = 0; + long usableSpace = 0; + for (File location : locations) { + usableSpace += location.getUsableSpace(); } return new ByteSizeValue(usableSpace); } @Override public boolean canDeleteUnallocated(ShardId shardId) { - if (location == null) { + if (locations == null) { return false; } if (indexService.hasShard(shardId.id())) { return false; } - return shardLocation(shardId).exists(); + for (File location : shardLocations(shardId)) { + if (location.exists()) { + return true; + } + } + return false; } @Override public void deleteUnallocated(ShardId shardId) throws IOException { - if (location == null) { + if (locations == null) { return; } if (indexService.hasShard(shardId.id())) { throw new ElasticSearchIllegalStateException(shardId + " allocated, can't be deleted"); } - FileSystemUtils.deleteRecursively(shardLocation(shardId)); + FileSystemUtils.deleteRecursively(shardLocations(shardId)); } - public File shardLocation(ShardId shardId) { - return nodeEnv.shardLocation(shardId); + public File[] shardLocations(ShardId shardId) { + return nodeEnv.shardLocations(shardId); } - public File shardIndexLocation(ShardId shardId) { - return new File(shardLocation(shardId), "index"); + public File[] shardIndexLocations(ShardId shardId) { + File[] shardLocations = shardLocations(shardId); + File[] shardIndexLocations = new File[shardLocations.length]; + for (int i = 0; i < shardLocations.length; i++) { + shardIndexLocations[i] = new File(shardLocations[i], "index"); + } + return shardIndexLocations; } // not used currently, but here to state that this store also defined a file based translog location - public File shardTranslogLocation(ShardId shardId) { - return new File(shardLocation(shardId), "translog"); + public File[] shardTranslogLocations(ShardId shardId) { + File[] shardLocations = shardLocations(shardId); + File[] shardTranslogLocations = new File[shardLocations.length]; + for (int i = 0; i < shardLocations.length; i++) { + shardTranslogLocations[i] = new File(shardLocations[i], "translog"); + } + return shardTranslogLocations; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java index c54066fc16a06..7a12ca5754ecd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java @@ -40,8 +40,12 @@ public class MmapFsDirectoryService extends FsDirectoryService { } @Override public Directory[] build() throws IOException { - File location = indexStore.shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - return new Directory[]{new MMapDirectory(location, buildLockFactory())}; + File[] locations = indexStore.shardIndexLocations(shardId); + Directory[] dirs = new Directory[locations.length]; + for (int i = 0; i < dirs.length; i++) { + FileSystemUtils.mkdirs(locations[i]); + dirs[i] = new MMapDirectory(locations[i], buildLockFactory()); + } + return dirs; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java index e020b9a349e8a..5f1efb09c223c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java @@ -40,8 +40,12 @@ public class NioFsDirectoryService extends FsDirectoryService { } @Override public Directory[] build() throws IOException { - File location = indexStore.shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - return new Directory[]{new NIOFSDirectory(location, buildLockFactory())}; + File[] locations = indexStore.shardIndexLocations(shardId); + Directory[] dirs = new Directory[locations.length]; + for (int i = 0; i < dirs.length; i++) { + FileSystemUtils.mkdirs(locations[i]); + dirs[i] = new NIOFSDirectory(locations[i], buildLockFactory()); + } + return dirs; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java index dee62bf263714..16514438bc4b0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java @@ -40,8 +40,12 @@ public class SimpleFsDirectoryService extends FsDirectoryService { } @Override public Directory[] build() throws IOException { - File location = indexStore.shardIndexLocation(shardId); - FileSystemUtils.mkdirs(location); - return new Directory[]{new SimpleFSDirectory(location, buildLockFactory())}; + File[] locations = indexStore.shardIndexLocations(shardId); + Directory[] dirs = new Directory[locations.length]; + for (int i = 0; i < dirs.length; i++) { + FileSystemUtils.mkdirs(locations[i]); + dirs[i] = new SimpleFSDirectory(locations[i], buildLockFactory()); + } + return dirs; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index 324713782fdd8..5489ab6824f6a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.CachedStreamOutput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.settings.IndexSettings; import org.elasticsearch.index.shard.AbstractIndexShardComponent; @@ -44,7 +45,7 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog { private final ReadWriteLock rwl = new ReentrantReadWriteLock(); - private final File location; + private final File[] locations; private volatile FsTranslogFile current; private volatile FsTranslogFile trans; @@ -53,18 +54,22 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog @Inject public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) { super(shardId, indexSettings); - this.location = new File(nodeEnv.shardLocation(shardId), "translog"); - FileSystemUtils.mkdirs(this.location); + File[] shardLocations = nodeEnv.shardLocations(shardId); + this.locations = new File[shardLocations.length]; + for (int i = 0; i < shardLocations.length; i++) { + locations[i] = new File(shardLocations[i], "translog"); + FileSystemUtils.mkdirs(locations[i]); + } } public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, File location) { super(shardId, indexSettings); - this.location = location; - FileSystemUtils.mkdirs(this.location); + this.locations = new File[]{location}; + FileSystemUtils.mkdirs(location); } - public File location() { - return location; + public File[] locations() { + return locations; } @Override public long currentId() { @@ -98,19 +103,21 @@ public File location() { @Override public void clearUnreferenced() { rwl.writeLock().lock(); try { - File[] files = location.listFiles(); - if (files != null) { - for (File file : files) { - if (file.getName().equals("translog-" + current.id())) { - continue; - } - if (trans != null && file.getName().equals("translog-" + trans.id())) { - continue; - } - try { - file.delete(); - } catch (Exception e) { - // ignore + for (File location : locations) { + File[] files = location.listFiles(); + if (files != null) { + for (File file : files) { + if (file.getName().equals("translog-" + current.id())) { + continue; + } + if (trans != null && file.getName().equals("translog-" + trans.id())) { + continue; + } + try { + file.delete(); + } catch (Exception e) { + // ignore + } } } } @@ -123,6 +130,17 @@ public File location() { rwl.writeLock().lock(); try { FsTranslogFile newFile; + long size = Long.MAX_VALUE; + File location = null; + for (File file : locations) { + long currentFree = file.getFreeSpace(); + if (currentFree < size) { + size = currentFree; + location = file; + } else if (currentFree == size && ThreadLocalRandom.current().nextBoolean()) { + location = file; + } + } try { newFile = new FsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); } catch (IOException e) { @@ -147,6 +165,17 @@ public File location() { rwl.writeLock().lock(); try { assert this.trans == null; + long size = Long.MAX_VALUE; + File location = null; + for (File file : locations) { + long currentFree = file.getFreeSpace(); + if (currentFree < size) { + size = currentFree; + location = file; + } else if (currentFree == size && ThreadLocalRandom.current().nextBoolean()) { + location = file; + } + } this.trans = new FsTranslogFile(shardId, id, new RafReference(new File(location, "translog-" + id))); } catch (IOException e) { throw new TranslogException(shardId, "failed to create new translog file", e); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java index 1ac25b53553e5..44c1a66cd2b57 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java @@ -352,7 +352,7 @@ private void deleteIndex(String index, boolean delete, String reason, @Nullable indicesLifecycle.afterIndexClosed(indexService.index(), delete); if (delete) { - FileSystemUtils.deleteRecursively(nodeEnv.indexLocation(new Index(index))); + FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(index))); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index f140a2abed5a0..24dfa3456c9bc 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -131,25 +131,28 @@ public void close() { } if (shardCanBeDeleted) { ShardId shardId = indexShardRoutingTable.shardId(); - File shardLocation = nodeEnv.shardLocation(shardId); - if (shardLocation.exists()) { - logger.debug("[{}][{}] deleting shard that is no longer used", shardId.index().name(), shardId.id()); - FileSystemUtils.deleteRecursively(shardLocation); + for (File shardLocation : nodeEnv.shardLocations(shardId)) { + if (shardLocation.exists()) { + logger.debug("[{}][{}] deleting shard that is no longer used", shardId.index().name(), shardId.id()); + FileSystemUtils.deleteRecursively(shardLocation); + } } } } } // delete indices that are no longer part of the metadata - File[] files = nodeEnv.indicesLocation().listFiles(); - if (files != null) { - for (File file : files) { - // if we have the index on the metadata, don't delete it - if (event.state().metaData().hasIndex(file.getName())) { - continue; + for (File indicesLocation : nodeEnv.indicesLocations()) { + File[] files = indicesLocation.listFiles(); + if (files != null) { + for (File file : files) { + // if we have the index on the metadata, don't delete it + if (event.state().metaData().hasIndex(file.getName())) { + continue; + } + logger.debug("[{}] deleting index that is no longer in the cluster meta_date from [{}]", file.getName(), file); + FileSystemUtils.deleteRecursively(file); } - logger.debug("[{}] deleting index that is no longer in the cluster meta_date", file.getName()); - FileSystemUtils.deleteRecursively(file); } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java index a25624cf18414..c36b31536ee19 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java @@ -19,7 +19,6 @@ package org.elasticsearch.indices.store; -import org.apache.lucene.store.FSDirectory; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.FailedNodeException; @@ -33,12 +32,10 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.collect.Maps; import org.elasticsearch.common.inject.Inject; -import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -163,17 +160,34 @@ private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException if (!storeType.contains("fs")) { return new StoreFilesMetaData(false, shardId, ImmutableMap.of()); } - File indexFile = new File(nodeEnv.shardLocation(shardId), "index"); - if (!indexFile.exists()) { + File[] shardLocations = nodeEnv.shardLocations(shardId); + File[] shardIndexLocations = new File[shardLocations.length]; + for (int i = 0; i < shardLocations.length; i++) { + shardIndexLocations[i] = new File(shardLocations[i], "index"); + } + boolean exists = false; + for (File shardIndexLocation : shardIndexLocations) { + if (shardIndexLocation.exists()) { + exists = true; + break; + } + } + if (!exists) { return new StoreFilesMetaData(false, shardId, ImmutableMap.of()); } + + Map checksums = Store.readChecksums(shardIndexLocations); + if (checksums == null) { + checksums = ImmutableMap.of(); + } + Map files = Maps.newHashMap(); - // read the checksums file - FSDirectory directory = FSDirectory.open(indexFile); - Map checksums = null; - try { - checksums = Store.readChecksums(directory); - for (File file : indexFile.listFiles()) { + for (File shardIndexLocation : shardIndexLocations) { + File[] listedFiles = shardIndexLocation.listFiles(); + if (listedFiles == null) { + continue; + } + for (File file : listedFiles) { // BACKWARD CKS SUPPORT if (file.getName().endsWith(".cks")) { continue; @@ -183,28 +197,6 @@ private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException } files.put(file.getName(), new StoreFileMetaData(file.getName(), file.length(), file.lastModified(), checksums.get(file.getName()))); } - } finally { - directory.close(); - } - - // BACKWARD CKS SUPPORT - for (File file : indexFile.listFiles()) { - if (file.getName().endsWith(".cks")) { - continue; - } - if (file.getName().startsWith("_checksums")) { - continue; - } - // try and load the checksum - String checksum = null; - File checksumFile = new File(file.getParentFile(), file.getName() + ".cks"); - if (checksumFile.exists() && (checksums == null || !checksums.containsKey(file.getName()))) { - byte[] checksumBytes = Streams.copyToByteArray(checksumFile); - if (checksumBytes.length > 0) { - checksum = Unicode.fromBytes(checksumBytes); - } - files.put(file.getName(), new StoreFileMetaData(file.getName(), file.length(), file.lastModified(), checksum)); - } } return new StoreFilesMetaData(false, shardId, files); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalSettingsPerparer.java b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalSettingsPerparer.java index 50194a4a91363..735a30db7ee68 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalSettingsPerparer.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalSettingsPerparer.java @@ -108,9 +108,11 @@ public static Tuple prepareSettings(Settings pSettings, b settingsBuilder = settingsBuilder().put(v1); settingsBuilder.put("path.home", cleanPath(environment.homeFile().getAbsolutePath())); settingsBuilder.put("path.work", cleanPath(environment.workFile().getAbsolutePath())); - settingsBuilder.put("path.work_with_cluster", cleanPath(environment.workWithClusterFile().getAbsolutePath())); - settingsBuilder.put("path.data", cleanPath(environment.dataFile().getAbsolutePath())); - settingsBuilder.put("path.data_with_cluster", cleanPath(environment.dataWithClusterFile().getAbsolutePath())); + String[] paths = new String[environment.dataFiles().length]; + for (int i = 0; i < environment.dataFiles().length; i++) { + paths[i] = cleanPath(environment.dataFiles()[i].getAbsolutePath()); + } + settingsBuilder.putArray("path.data", paths); settingsBuilder.put("path.logs", cleanPath(environment.logsFile().getAbsolutePath())); v1 = settingsBuilder.build(); diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/gateway/fs/AbstractSimpleIndexGatewayTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/gateway/fs/AbstractSimpleIndexGatewayTests.java index 3cccbebe0b0c6..a75ce07e12d04 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/gateway/fs/AbstractSimpleIndexGatewayTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/gateway/fs/AbstractSimpleIndexGatewayTests.java @@ -165,7 +165,7 @@ public abstract class AbstractSimpleIndexGatewayTests extends AbstractNodesTests logger.info("Closing the server"); closeNode("server1"); logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway"); - FileSystemUtils.deleteRecursively(environment.dataWithClusterFile()); + FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles()); logger.info("Starting the server, should recover from the gateway (both index and translog) without reusing work dir"); startNode("server1"); @@ -282,7 +282,7 @@ private void testLoad(boolean fullRecovery) { closeNode("server1"); if (fullRecovery) { logger.info("Clearing cluster data dir, so there will be a full recovery from the gateway"); - FileSystemUtils.deleteRecursively(environment.dataWithClusterFile()); + FileSystemUtils.deleteRecursively(environment.dataWithClusterFiles()); logger.info("Starting the server, should recover from the gateway (both index and translog) without reusing work dir"); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/store/IndicesStoreTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/store/IndicesStoreTests.java index 133ca1d831305..7a46104c97c63 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/store/IndicesStoreTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/store/IndicesStoreTests.java @@ -32,8 +32,7 @@ import java.io.File; import static org.elasticsearch.client.Requests.*; -import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; - +import static org.elasticsearch.common.settings.ImmutableSettings.*; import static org.hamcrest.MatcherAssert.*; import static org.hamcrest.Matchers.*; @@ -118,7 +117,7 @@ public void shardsCleanup() { private File shardDirectory(String server, String index, int shard) { InternalNode node = ((InternalNode) node(server)); NodeEnvironment env = node.injector().getInstance(NodeEnvironment.class); - return env.shardLocation(new ShardId(index, shard)); + return env.shardLocations(new ShardId(index, shard))[0]; } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java index ac8e60afadaa6..41d1b9588fc5f 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/fullrestart/FullRestartStressTest.java @@ -197,10 +197,10 @@ public void run() throws Exception { client.close(); for (Node node : nodes) { - File nodeWork = ((InternalNode) node).injector().getInstance(NodeEnvironment.class).nodeDataLocation(); + File[] nodeDatas = ((InternalNode) node).injector().getInstance(NodeEnvironment.class).nodeDataLocations(); node.close(); if (clearNodeWork && !settings.get("gateway.type").equals("local")) { - FileSystemUtils.deleteRecursively(nodeWork); + FileSystemUtils.deleteRecursively(nodeDatas); } } @@ -221,6 +221,7 @@ public static void main(String[] args) throws Exception { .put("gateway.type", "local") .put("gateway.recover_after_nodes", numberOfNodes) .put("index.number_of_shards", 1) + .put("path.data", "data/data1,data/data2") .build(); FullRestartStressTest test = new FullRestartStressTest() diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/rollingrestart/RollingRestartStressTest.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/rollingrestart/RollingRestartStressTest.java index ed730d00576fe..8e6fb3567a487 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/rollingrestart/RollingRestartStressTest.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/rollingrestart/RollingRestartStressTest.java @@ -167,7 +167,7 @@ public void run() throws Exception { // start doing the rolling restart int nodeIndex = 0; while (true) { - File nodeData = ((InternalNode) nodes[nodeIndex]).injector().getInstance(NodeEnvironment.class).nodeDataLocation(); + File[] nodeData = ((InternalNode) nodes[nodeIndex]).injector().getInstance(NodeEnvironment.class).nodeDataLocations(); nodes[nodeIndex].close(); if (clearNodeData) { FileSystemUtils.deleteRecursively(nodeData); @@ -310,7 +310,7 @@ private class Indexer extends Thread { } private void indexDoc() throws Exception { - StringBuffer sb = new StringBuffer(); + StringBuilder sb = new StringBuilder(); XContentBuilder json = XContentFactory.jsonBuilder().startObject() .field("field", "value" + ThreadLocalRandom.current().nextInt()); @@ -341,6 +341,7 @@ public static void main(String[] args) throws Exception { Settings settings = settingsBuilder() .put("index.shard.check_index", true) .put("gateway.type", "none") + .put("path.data", "data/data1,data/data2") .build(); RollingRestartStressTest test = new RollingRestartStressTest() From d76d7d4a563624f4b947b5fc92a4cdd3c27bd0ed Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 23 Sep 2011 00:56:02 +0300 Subject: [PATCH 54/96] Support external versioning for deletes arriving before initial update, closes #1351. --- .../index/engine/robin/RobinEngine.java | 3 ++- .../versioning/SimpleVersioningTests.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 241ec97db9de2..064657cd1e097 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -634,7 +634,8 @@ private void innerDelete(Delete delete, IndexWriter writer) throws IOException { updatedVersion = currentVersion < 0 ? 1 : currentVersion + 1; } else { // External if (currentVersion == -1) { - throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); + // its an external version, that's fine, we allow it to be set + //throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), -1, delete.version()); } else if (currentVersion >= delete.version()) { throw new VersionConflictEngineException(shardId, delete.type(), delete.id(), currentVersion, delete.version()); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java index de9f1ab00f9d5..6ee70fd43c764 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/versioning/SimpleVersioningTests.java @@ -61,6 +61,24 @@ public class SimpleVersioningTests extends AbstractNodesTests { closeAllNodes(); } + @Test public void testExternalVersioningInitialDelete() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test").execute().actionGet(); + client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); + + DeleteResponse deleteResponse = client2.prepareDelete("test", "type", "1").setVersion(17).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + assertThat(deleteResponse.notFound(), equalTo(true)); + + try { + client.prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(13).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + } catch (ElasticSearchException e) { + assertThat(e.unwrapCause(), instanceOf(VersionConflictEngineException.class)); + } + + client.prepareIndex("test", "type", "1").setSource("field1", "value1_1").setVersion(18).setVersionType(VersionType.EXTERNAL).execute().actionGet(); + } + @Test public void testExternalVersioning() throws Exception { try { client.admin().indices().prepareDelete("test").execute().actionGet(); From 49d976fc4113917024ea65604627d9d4f7ab0736 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 23 Sep 2011 00:57:57 +0300 Subject: [PATCH 55/96] add a commented out option to force the JVM to use IPv4 stack --- bin/elasticsearch.in.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/elasticsearch.in.sh b/bin/elasticsearch.in.sh index 56179a794e35c..b3230b5818473 100644 --- a/bin/elasticsearch.in.sh +++ b/bin/elasticsearch.in.sh @@ -17,6 +17,9 @@ JAVA_OPTS="$JAVA_OPTS -Xmx${ES_MAX_MEM}" # reduce the per-thread stack size JAVA_OPTS="$JAVA_OPTS -Xss128k" +# Force the JVM to use IPv4 stack +# JAVA_OPTS="$JAVA_OPTS -Djava.net.preferIPv4Stack=true" + # Enable aggressive optimizations in the JVM # - Disabled by default as it might cause the JVM to crash # JAVA_OPTS="$JAVA_OPTS -XX:+AggressiveOpts" From f74793c92a544478ba240c93f811a3a58488aa2a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 23 Sep 2011 17:08:21 +0300 Subject: [PATCH 56/96] improve multiple data locations when reading checksums and local gateway state files by going through all the locations to find them. --- .../gateway/local/LocalGateway.java | 180 +++++++++++------- .../org/elasticsearch/index/store/Store.java | 90 ++++----- 2 files changed, 148 insertions(+), 122 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index 1a96c94949517..0cba6785f7de8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -73,7 +73,7 @@ */ public class LocalGateway extends AbstractLifecycleComponent implements Gateway, ClusterStateListener { - private File location; + private boolean requiresStatePersistence; private final ClusterService clusterService; @@ -184,8 +184,7 @@ public LocalGatewayStartedShards currentStartedShards() { } @Override public void clusterChanged(final ClusterChangedEvent event) { - // the location is set to null, so we should not store it (for example, its not a data/master node) - if (location == null) { + if (!requiresStatePersistence) { return; } @@ -260,20 +259,18 @@ private synchronized void lazyInitialize() { // if this is not a possible master node or data node, bail, we won't save anything here... if (!clusterService.localNode().masterNode() && !clusterService.localNode().dataNode()) { - location = null; + requiresStatePersistence = false; } else { // create the location where the state will be stored // TODO: we might want to persist states on all data locations - this.location = new File(nodeEnv.nodeDataLocations()[0], "_state"); - FileSystemUtils.mkdirs(this.location); + requiresStatePersistence = true; if (clusterService.localNode().masterNode()) { try { - long version = findLatestMetaStateVersion(); - if (version != -1) { - File file = new File(location, "metadata-" + version); - logger.debug("[find_latest_state]: loading metadata from [{}]", file.getAbsolutePath()); - this.currentMetaState = readMetaState(Streams.copyToByteArray(new FileInputStream(file))); + File latest = findLatestMetaStateVersion(); + if (latest != null) { + logger.debug("[find_latest_state]: loading metadata from [{}]", latest.getAbsolutePath()); + this.currentMetaState = readMetaState(Streams.copyToByteArray(new FileInputStream(latest))); } else { logger.debug("[find_latest_state]: no metadata state loaded"); } @@ -284,11 +281,10 @@ private synchronized void lazyInitialize() { if (clusterService.localNode().dataNode()) { try { - long version = findLatestStartedShardsVersion(); - if (version != -1) { - File file = new File(location, "shards-" + version); - logger.debug("[find_latest_state]: loading started shards from [{}]", file.getAbsolutePath()); - this.currentStartedShards = readStartedShards(Streams.copyToByteArray(new FileInputStream(file))); + File latest = findLatestStartedShardsVersion(); + if (latest != null) { + logger.debug("[find_latest_state]: loading started shards from [{}]", latest.getAbsolutePath()); + this.currentStartedShards = readStartedShards(Streams.copyToByteArray(new FileInputStream(latest))); } else { logger.debug("[find_latest_state]: no started shards loaded"); } @@ -299,63 +295,85 @@ private synchronized void lazyInitialize() { } } - private long findLatestStartedShardsVersion() throws IOException { + private File findLatestStartedShardsVersion() throws IOException { long index = -1; - for (File stateFile : location.listFiles()) { - if (logger.isTraceEnabled()) { - logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + File latest = null; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; } - String name = stateFile.getName(); - if (!name.startsWith("shards-")) { + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { continue; } - long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); - if (fileIndex >= index) { - // try and read the meta data - try { - byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); - if (data.length == 0) { - logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); + for (File stateFile : stateFiles) { + if (logger.isTraceEnabled()) { + logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + } + String name = stateFile.getName(); + if (!name.startsWith("shards-")) { + continue; + } + long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); + if (fileIndex >= index) { + // try and read the meta data + try { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); + } + readStartedShards(data); + index = fileIndex; + latest = stateFile; + } catch (IOException e) { + logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); } - readStartedShards(data); - index = fileIndex; - } catch (IOException e) { - logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); } } } - - return index; + return latest; } - private long findLatestMetaStateVersion() throws IOException { + private File findLatestMetaStateVersion() throws IOException { long index = -1; - for (File stateFile : location.listFiles()) { - if (logger.isTraceEnabled()) { - logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + File latest = null; + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; } - String name = stateFile.getName(); - if (!name.startsWith("metadata-")) { + File[] stateFiles = stateLocation.listFiles(); + if (stateFiles == null) { continue; } - long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); - if (fileIndex >= index) { - // try and read the meta data - try { - byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); - if (data.length == 0) { - logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); - continue; + for (File stateFile : stateFiles) { + if (logger.isTraceEnabled()) { + logger.trace("[find_latest_state]: processing [" + stateFile.getName() + "]"); + } + String name = stateFile.getName(); + if (!name.startsWith("metadata-")) { + continue; + } + long fileIndex = Long.parseLong(name.substring(name.indexOf('-') + 1)); + if (fileIndex >= index) { + // try and read the meta data + try { + byte[] data = Streams.copyToByteArray(new FileInputStream(stateFile)); + if (data.length == 0) { + logger.debug("[find_latest_state]: not data for [" + name + "], ignoring..."); + continue; + } + readMetaState(data); + index = fileIndex; + latest = stateFile; + } catch (IOException e) { + logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); } - readMetaState(data); - index = fileIndex; - } catch (IOException e) { - logger.warn("[find_latest_state]: failed to read state from [" + name + "], ignoring...", e); } } } - - return index; + return latest; } private LocalGatewayMetaState readMetaState(byte[] data) throws IOException { @@ -411,7 +429,11 @@ public PersistMetaData(ClusterChangedEvent event) { builder.metaData(event.state().metaData()); try { - File stateFile = new File(location, "metadata-" + version); + File stateLocation = new File(nodeEnv.nodeDataLocations()[0], "_state"); + if (!stateLocation.exists()) { + FileSystemUtils.mkdirs(stateLocation); + } + File stateFile = new File(stateLocation, "metadata-" + version); OutputStream fos = new FileOutputStream(stateFile); if (compress) { fos = new LZFOutputStream(fos); @@ -432,14 +454,20 @@ public PersistMetaData(ClusterChangedEvent event) { currentMetaState = stateToWrite; // delete all the other files - File[] files = location.listFiles(new FilenameFilter() { - @Override public boolean accept(File dir, String name) { - return name.startsWith("metadata-") && !name.equals("metadata-" + version); + for (File dataLocation : nodeEnv.nodeDataLocations()) { + stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; } - }); - if (files != null) { - for (File file : files) { - file.delete(); + File[] files = stateLocation.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { + return name.startsWith("metadata-") && !name.equals("metadata-" + version); + } + }); + if (files != null) { + for (File file : files) { + file.delete(); + } } } @@ -461,7 +489,11 @@ public PersistShards(ClusterChangedEvent event, LocalGatewayStartedShards stateT @Override public void run() { try { - File stateFile = new File(location, "shards-" + event.state().version()); + File stateLocation = new File(nodeEnv.nodeDataLocations()[0], "_state"); + if (!stateLocation.exists()) { + FileSystemUtils.mkdirs(stateLocation); + } + File stateFile = new File(stateLocation, "shards-" + event.state().version()); OutputStream fos = new FileOutputStream(stateFile); if (compress) { fos = new LZFOutputStream(fos); @@ -487,14 +519,20 @@ public PersistShards(ClusterChangedEvent event, LocalGatewayStartedShards stateT } // delete all the other files - File[] files = location.listFiles(new FilenameFilter() { - @Override public boolean accept(File dir, String name) { - return name.startsWith("shards-") && !name.equals("shards-" + event.state().version()); + for (File dataLocation : nodeEnv.nodeDataLocations()) { + File stateLocation = new File(dataLocation, "_state"); + if (!stateLocation.exists()) { + continue; } - }); - if (files != null) { - for (File file : files) { - file.delete(); + File[] files = stateLocation.listFiles(new FilenameFilter() { + @Override public boolean accept(File dir, String name) { + return name.startsWith("shards-") && !name.equals("shards-" + event.state().version()); + } + }); + if (files != null) { + for (File file : files) { + file.delete(); + } } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java index d890a03b36516..0ea837aaca487 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java @@ -25,8 +25,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.SimpleFSDirectory; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.Unicode; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.MapBuilder; @@ -164,35 +164,44 @@ public void renameFile(String from, String to) throws IOException { } public static Map readChecksums(File[] locations) throws IOException { - for (File location : locations) { - FSDirectory directory = FSDirectory.open(location); - try { - Map checksums = readChecksums(directory, null); - if (checksums != null) { - return checksums; + Directory[] dirs = new Directory[locations.length]; + try { + for (int i = 0; i < locations.length; i++) { + dirs[i] = new SimpleFSDirectory(locations[i]); + } + return readChecksums(dirs, null); + } finally { + for (Directory dir : dirs) { + if (dir != null) { + try { + dir.close(); + } catch (IOException e) { + // ignore + } } - } finally { - directory.close(); } } - return null; } - static Map readChecksums(Directory dir, Map defaultValue) throws IOException { + static Map readChecksums(Directory[] dirs, Map defaultValue) throws IOException { long lastFound = -1; - for (String name : dir.listAll()) { - if (!isChecksum(name)) { - continue; - } - long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); - if (current > lastFound) { - lastFound = current; + Directory lastDir = null; + for (Directory dir : dirs) { + for (String name : dir.listAll()) { + if (!isChecksum(name)) { + continue; + } + long current = Long.parseLong(name.substring(CHECKSUMS_PREFIX.length())); + if (current > lastFound) { + lastFound = current; + lastDir = dir; + } } } if (lastFound == -1) { return defaultValue; } - IndexInput indexInput = dir.openInput(CHECKSUMS_PREFIX + lastFound); + IndexInput indexInput = lastDir.openInput(CHECKSUMS_PREFIX + lastFound); try { indexInput.readInt(); // version return indexInput.readStringStringMap(); @@ -205,10 +214,6 @@ static Map readChecksums(Directory dir, Map defa } public void writeChecksums() throws IOException { - writeChecksums(directory); - } - - private void writeChecksums(StoreDirectory dir) throws IOException { String checksumName = CHECKSUMS_PREFIX + System.currentTimeMillis(); ImmutableMap files = list(); synchronized (mutex) { @@ -218,7 +223,7 @@ private void writeChecksums(StoreDirectory dir) throws IOException { checksums.put(metaData.name(), metaData.checksum()); } } - IndexOutput output = dir.createOutput(checksumName, false); + IndexOutput output = directory.createOutput(checksumName, false); output.writeInt(0); // version output.writeStringStringMap(checksums); output.close(); @@ -226,7 +231,7 @@ private void writeChecksums(StoreDirectory dir) throws IOException { for (StoreFileMetaData metaData : files.values()) { if (metaData.name().startsWith(CHECKSUMS_PREFIX) && !checksumName.equals(metaData.name())) { try { - dir.deleteFileChecksum(metaData.name()); + directory.deleteFileChecksum(metaData.name()); } catch (Exception e) { // ignore } @@ -282,30 +287,10 @@ protected class StoreDirectory extends Directory implements ForceSyncDirectory { this.delegates = delegates; synchronized (mutex) { MapBuilder builder = MapBuilder.newMapBuilder(); - Map checksums = readChecksums(delegates[0], new HashMap()); + Map checksums = readChecksums(delegates, new HashMap()); for (Directory delegate : delegates) { for (String file : delegate.listAll()) { - // BACKWARD CKS SUPPORT - if (file.endsWith(".cks")) { // ignore checksum files here - continue; - } String checksum = checksums.get(file); - - // BACKWARD CKS SUPPORT - if (checksum == null) { - if (delegate.fileExists(file + ".cks")) { - IndexInput indexInput = delegate.openInput(file + ".cks"); - try { - if (indexInput.length() > 0) { - byte[] checksumBytes = new byte[(int) indexInput.length()]; - indexInput.readBytes(checksumBytes, 0, checksumBytes.length, false); - checksum = Unicode.fromBytes(checksumBytes); - } - } finally { - indexInput.close(); - } - } - } builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), checksum, delegate)); } } @@ -350,11 +335,14 @@ public Directory[] delegates() { } public void deleteFileChecksum(String name) throws IOException { - try { - delegates[0].deleteFile(name); - } catch (IOException e) { - if (delegates[0].fileExists(name)) { - throw e; + StoreFileMetaData metaData = filesMetadata.get(name); + if (metaData != null) { + try { + metaData.directory().deleteFile(name); + } catch (IOException e) { + if (metaData.directory().fileExists(name)) { + throw e; + } } } synchronized (mutex) { From d9d452a1ef15f8b37677544ae6f1c4672462cef5 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Fri, 23 Sep 2011 17:35:31 +0300 Subject: [PATCH 57/96] Allow to disable shard allocations, closes #1358. --- .../decider/AllocationDeciders.java | 1 + .../decider/AllocationDecidersModule.java | 1 + .../decider/DisableAllocationDecider.java | 77 +++++++++++++ .../allocation/DisableAllocationTests.java | 107 ++++++++++++++++++ 4 files changed, 186 insertions(+) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/DisableAllocationTests.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 551ed870dec9e..36e29614cbc2d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -45,6 +45,7 @@ public AllocationDeciders(Settings settings, NodeSettingsService nodeSettingsSer .add(new RebalanceOnlyWhenActiveAllocationDecider(settings)) .add(new ClusterRebalanceAllocationDecider(settings)) .add(new ConcurrentRebalanceAllocationDecider(settings)) + .add(new DisableAllocationDecider(settings, nodeSettingsService)) .add(new AwarenessAllocationDecider(settings, nodeSettingsService)) .build() ); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java index 675fbe8bf1faf..2bece7079e82a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDecidersModule.java @@ -47,6 +47,7 @@ public AllocationDecidersModule(Settings settings) { allocationMultibinder.addBinding().to(RebalanceOnlyWhenActiveAllocationDecider.class); allocationMultibinder.addBinding().to(ClusterRebalanceAllocationDecider.class); allocationMultibinder.addBinding().to(ConcurrentRebalanceAllocationDecider.class); + allocationMultibinder.addBinding().to(DisableAllocationDecider.class); allocationMultibinder.addBinding().to(AwarenessAllocationDecider.class); for (Class allocation : allocations) { allocationMultibinder.addBinding().to(allocation); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java new file mode 100644 index 0000000000000..5e6acf73d7018 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DisableAllocationDecider.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.settings.NodeSettingsService; + +/** + */ +public class DisableAllocationDecider extends AllocationDecider { + + static { + MetaData.addDynamicSettings( + "cluster.routing.allocation.disable_allocation", + "cluster.routing.allocation.disable_replica_allocation" + ); + } + + class ApplySettings implements NodeSettingsService.Listener { + @Override public void onRefreshSettings(Settings settings) { + boolean disableAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_allocation", DisableAllocationDecider.this.disableAllocation); + if (disableAllocation != DisableAllocationDecider.this.disableAllocation) { + logger.info("updating [cluster.routing.allocation.disable_allocation] from [{}] to [{}]", DisableAllocationDecider.this.disableAllocation, disableAllocation); + DisableAllocationDecider.this.disableAllocation = disableAllocation; + } + + boolean disableReplicaAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_replica_allocation", DisableAllocationDecider.this.disableReplicaAllocation); + if (disableReplicaAllocation != DisableAllocationDecider.this.disableReplicaAllocation) { + logger.info("updating [cluster.routing.allocation.disable_replica_allocation] from [{}] to [{}]", DisableAllocationDecider.this.disableReplicaAllocation, disableReplicaAllocation); + DisableAllocationDecider.this.disableReplicaAllocation = disableReplicaAllocation; + } + } + } + + private volatile boolean disableAllocation; + private volatile boolean disableReplicaAllocation; + + @Inject public DisableAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { + super(settings); + this.disableAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_allocation", false); + this.disableReplicaAllocation = settings.getAsBoolean("cluster.routing.allocation.disable_replica_allocation", false); + + nodeSettingsService.addListener(new ApplySettings()); + } + + @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + if (disableAllocation) { + return Decision.NO; + } + if (disableReplicaAllocation) { + return shardRouting.primary() ? Decision.YES : Decision.NO; + } + return Decision.YES; + } +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/DisableAllocationTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/DisableAllocationTests.java new file mode 100644 index 0000000000000..18ed1c0a93666 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/cluster/routing/allocation/DisableAllocationTests.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.testng.annotations.Test; + +import static org.elasticsearch.cluster.ClusterState.*; +import static org.elasticsearch.cluster.metadata.IndexMetaData.*; +import static org.elasticsearch.cluster.metadata.MetaData.*; +import static org.elasticsearch.cluster.node.DiscoveryNodes.*; +import static org.elasticsearch.cluster.routing.RoutingBuilders.*; +import static org.elasticsearch.cluster.routing.ShardRoutingState.*; +import static org.elasticsearch.cluster.routing.allocation.RoutingAllocationTests.*; +import static org.elasticsearch.common.settings.ImmutableSettings.*; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + */ +@Test +public class DisableAllocationTests { + + private final ESLogger logger = Loggers.getLogger(DisableAllocationTests.class); + + @Test public void testDisableAllocation() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.disable_allocation", true) + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1")) + .put(newNode("node2")) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); + + } + + @Test public void testDisableReplicaAllocation() { + AllocationService strategy = new AllocationService(settingsBuilder() + .put("cluster.routing.allocation.disable_replica_allocation", true) + .build()); + + logger.info("Building initial routing table"); + + MetaData metaData = newMetaDataBuilder() + .put(newIndexMetaDataBuilder("test").numberOfShards(1).numberOfReplicas(1)) + .build(); + + RoutingTable routingTable = routingTable() + .add(indexRoutingTable("test").initializeEmpty(metaData.index("test"))) + .build(); + + ClusterState clusterState = newClusterStateBuilder().metaData(metaData).routingTable(routingTable).build(); + + logger.info("--> adding two nodes on same rack and do rerouting"); + clusterState = newClusterStateBuilder().state(clusterState).nodes(newNodesBuilder() + .put(newNode("node1")) + .put(newNode("node2")) + ).build(); + routingTable = strategy.reroute(clusterState).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(1)); + + logger.info("--> start the shards (primaries)"); + routingTable = strategy.applyStartedShards(clusterState, clusterState.routingNodes().shardsWithState(INITIALIZING)).routingTable(); + clusterState = newClusterStateBuilder().state(clusterState).routingTable(routingTable).build(); + + assertThat(clusterState.routingNodes().shardsWithState(INITIALIZING).size(), equalTo(0)); + } +} From d954a93d9ddd53d2e7fccb1caa543975caad9be7 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 24 Sep 2011 01:04:12 +0300 Subject: [PATCH 58/96] Malformed REST create index causes the index to still be created, closes #1359. --- .../admin/indices/create/RestCreateIndexAction.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java index bfa35f9f27a6d..6900a97cbb51d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/create/RestCreateIndexAction.java @@ -29,7 +29,12 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import org.elasticsearch.rest.action.support.RestXContentBuilder; import java.io.IOException; @@ -79,8 +84,8 @@ public class RestCreateIndexAction extends BaseRestHandler { channel.sendResponse(new XContentThrowableRestResponse(request, e)); } catch (IOException e1) { logger.warn("Failed to send response", e1); - return; } + return; } } else { // its plain settings, parse and set them @@ -91,8 +96,8 @@ public class RestCreateIndexAction extends BaseRestHandler { channel.sendResponse(new XContentThrowableRestResponse(request, BAD_REQUEST, new SettingsException("Failed to parse index settings", e))); } catch (IOException e1) { logger.warn("Failed to send response", e1); - return; } + return; } } } From 9d9133a451d3fbfe441d4aecb97a5be0e761f37e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 24 Sep 2011 01:59:21 +0300 Subject: [PATCH 59/96] required _routing fails when path points to an integer field, closes #1357. --- .../index/mapper/core/ByteFieldMapper.java | 4 ++++ .../index/mapper/core/DoubleFieldMapper.java | 4 ++++ .../index/mapper/core/FloatFieldMapper.java | 4 ++++ .../index/mapper/core/IntegerFieldMapper.java | 4 ++++ .../index/mapper/core/LongFieldMapper.java | 4 ++++ .../index/mapper/core/NumberFieldMapper.java | 2 ++ .../index/mapper/core/ShortFieldMapper.java | 4 ++++ .../mapper/internal/RoutingFieldMapper.java | 16 ++++++++++++- .../routing/SimpleRoutingTests.java | 24 +++++++++++++++++++ 9 files changed, 65 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java index 47abfb0bb4c53..529f97a4cbf09 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java @@ -307,5 +307,9 @@ public CustomByteNumericField(NumberFieldMapper mapper, byte number) { } return null; } + + @Override public String numericAsString() { + return Byte.toString(number); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java index b927db762942b..2542071c8ce00 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/DoubleFieldMapper.java @@ -308,5 +308,9 @@ public CustomDoubleNumericField(NumberFieldMapper mapper, double number) { } return null; } + + @Override public String numericAsString() { + return Double.toString(number); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java index ed68c1e62c4bb..ee3726bc9e5f7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/FloatFieldMapper.java @@ -304,5 +304,9 @@ public CustomFloatNumericField(NumberFieldMapper mapper, float number) { } return null; } + + @Override public String numericAsString() { + return Float.toString(number); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java index 0e053aefc4f25..0b7fca922eb13 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/IntegerFieldMapper.java @@ -308,5 +308,9 @@ public CustomIntegerNumericField(NumberFieldMapper mapper, int number) { } return null; } + + @Override public String numericAsString() { + return Integer.toString(number); + } } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java index ffa104cc813a3..8b5929d37247b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/LongFieldMapper.java @@ -307,5 +307,9 @@ public CustomLongNumericField(NumberFieldMapper mapper, long number) { } return null; } + + @Override public String numericAsString() { + return Long.toString(number); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java index f7be4fc34e560..cdb970dd354e1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java @@ -248,5 +248,7 @@ public CustomNumericField(NumberFieldMapper mapper, byte[] value) { @Override public Reader readerValue() { return null; } + + public abstract String numericAsString(); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java index b443f6b896d65..99e4f37c924db 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/ShortFieldMapper.java @@ -307,5 +307,9 @@ public CustomShortNumericField(NumberFieldMapper mapper, short number) { } return null; } + + @Override public String numericAsString() { + return Short.toString(number); + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index 420463fc2a31f..8da786aa809e2 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.RootMapper; import org.elasticsearch.index.mapper.core.AbstractFieldMapper; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; import java.io.IOException; import java.util.Map; @@ -156,10 +157,23 @@ public String value(Document document) { String routing = context.sourceToParse().routing(); if (path != null && routing != null) { // we have a path, check if we can validate we have the same routing value as the one in the doc... - String value = context.doc().get(path); + String value = null; + Fieldable field = context.doc().getFieldable(path); + if (field != null) { + value = field.stringValue(); + if (value == null) { + // maybe its a numeric field... + if (field instanceof NumberFieldMapper.CustomNumericField) { + value = ((NumberFieldMapper.CustomNumericField) field).numericAsString(); + } + } + } if (value == null) { value = context.ignoredValue(path); } + if (value == null) { + // maybe its a numeric field + } if (!routing.equals(value)) { throw new MapperParsingException("External routing [" + routing + "] and document path routing [" + value + "] mismatch"); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/routing/SimpleRoutingTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/routing/SimpleRoutingTests.java index 2c8344a324403..7df8cb273eac2 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/routing/SimpleRoutingTests.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/routing/SimpleRoutingTests.java @@ -300,4 +300,28 @@ protected Client getClient() { assertThat(client.prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().exists(), equalTo(true)); } } + + @Test public void testRequiredRoutingWithPathNumericType() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + + client.admin().indices().prepareCreate("test") + .addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("_routing").field("required", true).field("path", "routing_field").endObject() + .endObject().endObject()) + .execute().actionGet(); + client.admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + + logger.info("--> indexing with id [1], and routing [0]"); + client.prepareIndex("test", "type1", "1").setSource("field", "value1", "routing_field", 0).execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + logger.info("--> verifying get with no routing, should not find anything"); + for (int i = 0; i < 5; i++) { + assertThat(client.prepareGet("test", "type1", "1").execute().actionGet().exists(), equalTo(false)); + } + logger.info("--> verifying get with routing, should find"); + for (int i = 0; i < 5; i++) { + assertThat(client.prepareGet("test", "type1", "1").setRouting("0").execute().actionGet().exists(), equalTo(true)); + } + } } From d7e3c9538f379843bf4a353766a72ede597a59ac Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sat, 24 Sep 2011 16:57:14 +0300 Subject: [PATCH 60/96] remove unnecessary check --- .../index/mapper/internal/RoutingFieldMapper.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java index 8da786aa809e2..14b4688d1cb12 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/RoutingFieldMapper.java @@ -171,9 +171,6 @@ public String value(Document document) { if (value == null) { value = context.ignoredValue(path); } - if (value == null) { - // maybe its a numeric field - } if (!routing.equals(value)) { throw new MapperParsingException("External routing [" + routing + "] and document path routing [" + value + "] mismatch"); } From f837213d6f1ff4262c1a43d6ea7f15d0d91de271 Mon Sep 17 00:00:00 2001 From: Karel Minarik Date: Thu, 22 Sep 2011 15:44:12 +0200 Subject: [PATCH 61/96] Reformatted and amended the example configuration file Edited elasticsearch.yml: * Separated different sections (using headers) * Added more information about nodes configuration * Added more information about various index configurations and their effects * Added information about setting network and HTTP configuration * Reworded information on gateway, recovery, discovery The example configuration file should allow operations stuff to quickly get a sense of ElasticSearch features relevant for systems support, and to understand how to configure node, cluster, network and discovery settings. The aim here is to vaguely respect the most often changed configuration settings, while having some top-to-bottom conceptual integrity. Table of Contents: * Cluster * Node * Index * Paths * Memory * Network And HTTP * Gateway * Recovery Throttling * Discovery --- config/elasticsearch.yml | 344 ++++++++++++++++++++++++++++++++++----- 1 file changed, 302 insertions(+), 42 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index febbb8a86c77c..b106343ca551c 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -1,42 +1,302 @@ -# The cluster name -#cluster.name: elasticsearch - -# Path Settings -#path.conf: /path/to/conf -#path.data: /path/to/data -#path.work: /path/to/work -#path.logs: /path/to/logs -#path.plugins: /path/to/plugins - -# Force all memory to be locked, forcing the JVM to never swap -# When setting it, make sure ES_MIN_MEM and ES_MAX_MEM are set to the same value -# and that the machine has enough memory to allocate. -#bootstrap.mlockall: true - -# Gateway Settings -# Controls when to start the initial recovery process when starting a new cluster -# allowing for better reused of existing data during recovery. -#gateway.recover_after_nodes: 1 -#gateway.recover_after_time: 5m -#gateway.expected_nodes: 2 - -# Recovery Throttling -# The number of concurrent recoveries happening on a node -#cluster.routing.allocation.node_initial_primaries_recoveries: 4 -#cluster.routing.allocation.node_concurrent_recoveries: 2 -# Peer shard recovery size based throttling (set to 100mb for example to enable) -#indices.recovery.max_size_per_sec: 0 -# Number open concurrent recovery streams allows -#indices.recovery.concurrent_streams: 5 - -# Controls the minimum number of master eligible nodes this node should "see" -# in order to operate within the cluster. -# Set this to a higher value (2-4) when running more than 2 nodes in the cluster -#discovery.zen.minimum_master_nodes: 1 - -# The time to wait for ping responses from other nodes when doing node discovery -#discovery.zen.ping.timeout: 3s - -# Unicast Discovery (disable multicast) -#discovery.zen.ping.multicast.enabled: false -#discovery.zen.ping.unicast.hosts: ["host1", "host2"] +##################### ElasticSearch Configuration Example ##################### + +# This file contains an overview of various configuration settings, +# targeted at operations staff. Application developers should +# consult the guide at . +# +# The installation procedure is covered at +# . +# +# ElasticSearch comes with reasonable defaults for most settings, +# so you can try it out without bothering with configuration. +# +# Most of the time, these defaults are just fine for running a production +# cluster. If you're fine-tuning your cluster, or wondering about the +# effect of certain configuration option, please _do ask_ on the +# mailing list or IRC channel [http://elasticsearch.org/community]. + +# See +# for information on supported formats and syntax for the configuration file. + + +################################### Cluster ################################### + +# Cluster name identifies your cluster for auto-discovery. If you're running +# multiple clusters on the same network, make sure you're using unique names. +# +# cluster.name: elasticsearch + + +#################################### Node ##################################### + +# Node names are generated dynamically on startup, so you're relieved +# from configuring them manually. You can tie this node to a specific name: +# +# node.name: "Franz Kafka" + +# Every node can be configured to allow or deny being eligible as the master, +# and to allow or deny to store the data. +# +# Allow this node to be eligible as a master node (enabled by default): +# +# node.master: true +# +# Allow this node to store data (enabled by default): +# +# node.data: true + +# You can exploit these settings to design advanced cluster topologies. +# +# 1. You want this node to never become a master node, only to hold data. +# This will be the "workhorse" of your cluster. +# +# node.master: false +# node.data: true +# +# 2. You want this node to only serve as a master: to not store any data and +# to have free resources. This will be the "coordinator" of your cluster. +# +# node.master: true +# node.data: false +# +# 3. You want this node to be neither master nor data node, but +# to act as a "search load balancer" (fetching data from nodes, +# aggregating results, etc.) +# +# node.master: false +# node.data: false + +# Use the Cluster Health API [http://localhost:9200/_cluster/health], the +# Node Info API [http://localhost:9200/_cluster/nodes] or GUI tools +# such as and +# to inspect the cluster state. + + +#################################### Index #################################### + +# You can set a number of options (such as shard/replica options, mapping +# or analyzer definitions, translog settings, ...) for indices globally, +# in this file. +# +# Note, that it makes more sense to configure index settings specifically for +# a certain index, either when creating it or by using the index templates API. +# +# See and +# +# for more information. + +# Set the number of shards (splits) of an index (5 by default): +# +# index.number_of_shards: 5 + +# Set the number of replicas (additional copies) of an index (1 by default): +# +# index.number_of_replicas: 1 + +# Note, that for development on a local machine, with small indices, it usually +# makes sense to "disable" the distributed features: +# +# index.number_of_shards: 1 +# index.number_of_replicas: 0 + +# These settings directly affect the performance of index and search operations +# in your cluster. Assuming you have enough machines to hold shards and +# replicas, the rule of thumb is: +# +# 1. Having more *shards* enhances the _indexing_ performance and allows to +# _distribute_ a big index across machines. +# 2. Having more *replicas* enhances the _search_ performance and improves the +# cluster _availability_. +# +# The "number_of_shards" is a one-time setting for an index. +# +# The "number_of_replicas" can be increased or decreased anytime, +# by using the Index Update Settings API. +# +# ElasticSearch takes care about load balancing, relocating, gathering the +# results from nodes, etc. Experiment with different settings to fine-tune +# your setup. + +# Use the Index Status API () to inspect +# the index status. + + +#################################### Paths #################################### + +# Path to directory containing configuration (this file and logging.yml): +# +# path.conf: /path/to/conf + +# Path to directory where to store index data allocated for this node: +# +# path.data: /path/to/data + +# Path to temporary files: +# +# path.work: /path/to/work + +# Path to log files: +# +# path.logs: /path/to/logs + +# Path to where plugins are installed: +# +# path.plugins: /path/to/plugins + + +################################### Memory #################################### + +# ElasticSearch performs poorly when JVM starts swapping: you should ensure that +# it _never_ swaps. +# +# Set this property to true to lock the memory: +# +# bootstrap.mlockall: true + +# Make sure that the ES_MIN_MEM and ES_MAX_MEM environment variables are set +# to the same value, and that the machine has enough memory to allocate +# for ElasticSearch, leaving enough memory for the operating system itself. +# +# You should also make sure that the ElasticSearch process is allowed to lock +# the memory, eg. by using `ulimit -l unlimited`. + + +############################## Network And HTTP ############################### + +# ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens +# on port 9200 for HTTP traffic and on port 9300 for node-to-node communication. + +# Set the bind address specifically (IPv4 or IPv6): +# +# network.bind_host: 192.168.0.1 + +# Set the address other nodes will use to communicate with this node. If not +# set, it is automatically derived. It must point to an actual IP address. +# +# network.publish_host: 192.168.0.1 + +# Set both 'bind_host' and 'publish_host': +# +# network.host: 192.168.0.1 + +# Set a custom port for the node to node communication (9300 by default): +# +# transport.port: 9300 + +# Enable compression for all communication between (disabled by default): +# +# transport.tcp.compress: true + +# Set a custom port to listen for HTTP traffic: +# +# http.port: 9200 + +# Set a custom allowed content length: +# +# http.max_content_length: 100mb + +# Disable HTTP completely: +# +# http.enabled: false + + +################################### Gateway ################################### + +# The gateway allows for persisting the cluster state between full cluster +# restarts. Every change to the state (such as adding an index) will be stored +# in the gateway, and when the cluster starts up for the first time, +# it will read its state from the gateway. + +# There are several types of gateway implementations. For more information, +# see . + +# The default gateway type is the "local" gateway (recommended): +# +# gateway.type: local + +# Settings below control how and when to start the initial recovery process on +# a full cluster restart (to reuse as much local data as possible). + +# Allow recovery process after N nodes in a cluster are up: +# +# gateway.recover_after_nodes: 1 + +# Set the timeout to initiate the recovery process, once the N nodes +# from previous setting are up (accepts time value): +# +# gateway.recover_after_time: 5m + +# Set how many nodes are expected in this cluster. Once these N nodes +# are up, begin recovery process immediately: +# +# gateway.expected_nodes: 2 + + +############################# Recovery Throttling ############################# + +# These settings allow to control the process of shards allocation between +# nodes during initial recovery, replica allocation, rebalancing, +# or when adding and removing nodes. + +# Set the number of concurrent recoveries happening on a node: +# +# 1. During the initial recovery +# +# cluster.routing.allocation.node_initial_primaries_recoveries: 4 +# +# 2. During adding/removing nodes, rebalancing, etc +# +# cluster.routing.allocation.node_concurrent_recoveries: 2 + +# Set to throttle throughput when recovering (eg. 100mb, by default unlimited): +# +# indices.recovery.max_size_per_sec: 0 + +# Set to limit the number of open concurrent streams when +# recovering a shard from a peer: +# +# indices.recovery.concurrent_streams: 5 + + +################################## Discovery ################################## + +# Discovery infrastructure ensures nodes can be found within a cluster +# and master node is elected. Multicast discovery is the default. + +# Set to ensure a node sees N other master eligible nodes to be considered +# operational within the cluster. Set this option to a higher value (2-4) +# for large clusters: +# +# discovery.zen.minimum_master_nodes: 1 + +# Set the time to wait for ping responses from other nodes when discovering. +# Set this option to a higher value on a slow or congested network +# to minimize discovery failures: +# +# discovery.zen.ping.timeout: 3s + +# See +# for more information. + +# Unicast discovery allows to explicitly control which nodes will be used +# to discover the cluster. It can be used when multicast is not present, +# or to restrict the cluster communication-wise. +# +# 1. Disable multicast discovery (enabled by default): +# +# discovery.zen.ping.multicast.enabled: false +# +# 2. Configure an initial list of master nodes in the cluster +# to perform discovery when new nodes (master or data) are started: +# +# discovery.zen.ping.unicast.hosts: ["host1", "host2:port", "host3[portX-portY]"] + +# EC2 discovery allows to use AWS EC2 API in order to perform discovery. +# +# You have to install the cloud-aws plugin for enabling the EC2 discovery. +# +# See +# for more information. +# +# See +# for a step-by-step tutorial. From 4876e5a968757153fdc243f41b34ecb584b9f48c Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 21:06:12 +0300 Subject: [PATCH 62/96] doc multi path.data locations --- config/elasticsearch.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index b106343ca551c..83cb9798f533e 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -127,9 +127,15 @@ # # path.conf: /path/to/conf -# Path to directory where to store index data allocated for this node: +# Path to directory where to store index data allocated for this node. # # path.data: /path/to/data +# +# Can optionally include more than one location, causing data to be stripped across +# the locations (ala RAID 0) on a file level, favouring locations with most +# free space on creation. For example: +# +# path.data: /path/to/data1,/path/to/data2 # Path to temporary files: # From e8b88acbd3a215d784b7e0966373b33c14143fee Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 21:09:15 +0300 Subject: [PATCH 63/96] allow to dynamically set cluster.routing.allocation.cluster_concurrent_rebalance using cluster update settings API --- .../ConcurrentRebalanceAllocationDecider.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java index 93f8f89eba86d..ff76f68645948 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; +import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.routing.MutableShardRouting; import org.elasticsearch.cluster.routing.RoutingNode; import org.elasticsearch.cluster.routing.ShardRouting; @@ -26,10 +27,27 @@ import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.settings.NodeSettingsService; public class ConcurrentRebalanceAllocationDecider extends AllocationDecider { - private final int clusterConcurrentRebalance; + static { + MetaData.addDynamicSettings( + "cluster.routing.allocation.cluster_concurrent_rebalance" + ); + } + + class ApplySettings implements NodeSettingsService.Listener { + @Override public void onRefreshSettings(Settings settings) { + int clusterConcurrentRebalance = settings.getAsInt("cluster.routing.allocation.cluster_concurrent_rebalance", ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance); + if (clusterConcurrentRebalance != ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance) { + logger.info("updating [cluster.routing.allocation.cluster_concurrent_rebalance] from [{}], to [{}]", ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance, clusterConcurrentRebalance); + ConcurrentRebalanceAllocationDecider.this.clusterConcurrentRebalance = clusterConcurrentRebalance; + } + } + } + + private volatile int clusterConcurrentRebalance; @Inject public ConcurrentRebalanceAllocationDecider(Settings settings) { super(settings); From fca3adcd59d9665b37df6b6f57d2ab137616323b Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 21:09:41 +0300 Subject: [PATCH 64/96] allow to dynamically set cluster.routing.allocation.cluster_concurrent_rebalance using cluster update settings API --- .../decider/ConcurrentRebalanceAllocationDecider.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java index ff76f68645948..1a9bb313b351e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java @@ -49,10 +49,11 @@ class ApplySettings implements NodeSettingsService.Listener { private volatile int clusterConcurrentRebalance; - @Inject public ConcurrentRebalanceAllocationDecider(Settings settings) { + @Inject public ConcurrentRebalanceAllocationDecider(Settings settings, NodeSettingsService nodeSettingsService) { super(settings); this.clusterConcurrentRebalance = settings.getAsInt("cluster.routing.allocation.cluster_concurrent_rebalance", 2); logger.debug("using [cluster_concurrent_rebalance] with [{}]", clusterConcurrentRebalance); + nodeSettingsService.addListener(new ApplySettings()); } @Override public boolean canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) { From 9bea9029662460a13ebd704683a1f766b932bc38 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 21:12:17 +0300 Subject: [PATCH 65/96] allow to dynamically set cluster.routing.allocation.cluster_concurrent_rebalance using cluster update settings API --- .../cluster/routing/allocation/decider/AllocationDeciders.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java index 36e29614cbc2d..ccb6d8314e476 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/AllocationDeciders.java @@ -44,7 +44,7 @@ public AllocationDeciders(Settings settings, NodeSettingsService nodeSettingsSer .add(new ThrottlingAllocationDecider(settings, nodeSettingsService)) .add(new RebalanceOnlyWhenActiveAllocationDecider(settings)) .add(new ClusterRebalanceAllocationDecider(settings)) - .add(new ConcurrentRebalanceAllocationDecider(settings)) + .add(new ConcurrentRebalanceAllocationDecider(settings, nodeSettingsService)) .add(new DisableAllocationDecider(settings, nodeSettingsService)) .add(new AwarenessAllocationDecider(settings, nodeSettingsService)) .build() From f36d89c55410fdd6e431c05e7dee007b25200bbb Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 21:28:55 +0300 Subject: [PATCH 66/96] use index iteration over iterator --- .../allocation/decider/SameShardAllocationDecider.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java index b2b9d9381d360..b8c259c4a6dcb 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/SameShardAllocationDecider.java @@ -26,6 +26,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import java.util.List; + /** * An allocation strategy that does not allow for the same shard instance to be allocated on the same node. */ @@ -36,9 +38,10 @@ public class SameShardAllocationDecider extends AllocationDecider { } @Override public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { - for (MutableShardRouting current : node.shards()) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { // we do not allow for two shards of the same shard id to exists on the same node - if (current.shardId().equals(shardRouting.shardId())) { + if (shards.get(i).shardId().equals(shardRouting.shardId())) { return Decision.NO; } } From 18f15f0a6fda1c24fd3c296ab94bf9fe63bf9183 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 22:17:09 +0300 Subject: [PATCH 67/96] use index iteration over iterator --- .../elasticsearch/cluster/routing/RoutingNodes.java | 4 +++- .../decider/ClusterRebalanceAllocationDecider.java | 12 +++++++++--- .../ConcurrentRebalanceAllocationDecider.java | 7 +++++-- .../RebalanceOnlyWhenActiveAllocationDecider.java | 4 ++-- .../decider/ThrottlingAllocationDecider.java | 10 ++++++++-- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index fdedd008a4d80..014b2d23aeca4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -176,7 +176,9 @@ public TObjectIntHashMap nodesPerAttributesCounts(String attributeName) public MutableShardRouting findPrimaryForReplica(ShardRouting shard) { assert !shard.primary(); for (RoutingNode routingNode : nodesToShards.values()) { - for (MutableShardRouting shardRouting : routingNode) { + List shards = routingNode.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shardRouting = shards.get(i); if (shardRouting.shardId().equals(shard.shardId()) && shardRouting.primary()) { return shardRouting; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java index 159984e4da836..b4dcac950a830 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ClusterRebalanceAllocationDecider.java @@ -26,6 +26,8 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import java.util.List; + public class ClusterRebalanceAllocationDecider extends AllocationDecider { public static enum ClusterRebalanceType { @@ -49,7 +51,7 @@ public static enum ClusterRebalanceType { logger.warn("[cluster.routing.allocation.allow_rebalance] has a wrong value {}, defaulting to 'indices_all_active'", allowRebalance); type = ClusterRebalanceType.INDICES_ALL_ACTIVE; } - logger.debug("using [allow_rebalance] with [{}]", type.toString().toLowerCase()); + logger.debug("using [cluster.routing.allocation.allow_rebalance] with [{}]", type.toString().toLowerCase()); } @Override public boolean canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) { @@ -60,7 +62,9 @@ public static enum ClusterRebalanceType { } } for (RoutingNode node : allocation.routingNodes()) { - for (MutableShardRouting shard : node) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shard = shards.get(i); if (shard.primary() && !shard.active()) { return false; } @@ -73,7 +77,9 @@ public static enum ClusterRebalanceType { return false; } for (RoutingNode node : allocation.routingNodes()) { - for (MutableShardRouting shard : node) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shard = shards.get(i); if (!shard.active()) { return false; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java index 1a9bb313b351e..39b2981745189 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ConcurrentRebalanceAllocationDecider.java @@ -29,6 +29,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.settings.NodeSettingsService; +import java.util.List; + public class ConcurrentRebalanceAllocationDecider extends AllocationDecider { static { @@ -62,8 +64,9 @@ class ApplySettings implements NodeSettingsService.Listener { } int rebalance = 0; for (RoutingNode node : allocation.routingNodes()) { - for (MutableShardRouting shard : node) { - if (shard.state() == ShardRoutingState.RELOCATING) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + if (shards.get(i).state() == ShardRoutingState.RELOCATING) { rebalance++; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java index 32c6df5082daa..23dc2cd4a323d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/RebalanceOnlyWhenActiveAllocationDecider.java @@ -40,8 +40,8 @@ public class RebalanceOnlyWhenActiveAllocationDecider extends AllocationDecider List shards = allocation.routingNodes().shardsRoutingFor(shardRouting); // its ok to check for active here, since in relocation, a shard is split into two in routing // nodes, once relocating, and one initializing - for (ShardRouting allShard : shards) { - if (!allShard.active()) { + for (int i = 0; i < shards.size(); i++) { + if (!shards.get(i).active()) { return false; } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java index e0da196bae372..ff9b0fa1dad35 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ThrottlingAllocationDecider.java @@ -29,6 +29,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.settings.NodeSettingsService; +import java.util.List; + /** */ public class ThrottlingAllocationDecider extends AllocationDecider { @@ -65,7 +67,9 @@ public class ThrottlingAllocationDecider extends AllocationDecider { // primary is unassigned, means we are going to do recovery from gateway // count *just the primary* currently doing recovery on the node and check against concurrent_recoveries int primariesInRecovery = 0; - for (MutableShardRouting shard : node) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shard = shards.get(i); if (shard.state() == ShardRoutingState.INITIALIZING && shard.primary()) { primariesInRecovery++; } @@ -82,7 +86,9 @@ public class ThrottlingAllocationDecider extends AllocationDecider { // count the number of recoveries on the node, its for both target (INITIALIZING) and source (RELOCATING) int currentRecoveries = 0; - for (MutableShardRouting shard : node) { + List shards = node.shards(); + for (int i = 0; i < shards.size(); i++) { + MutableShardRouting shard = shards.get(i); if (shard.state() == ShardRoutingState.INITIALIZING || shard.state() == ShardRoutingState.RELOCATING) { currentRecoveries++; } From 32526f004cbbdaf949b2017d4e0135d6e5264d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20Brattberg?= Date: Sun, 25 Sep 2011 22:00:13 +0300 Subject: [PATCH 68/96] Minor spelling corrections --- config/elasticsearch.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 83cb9798f533e..666962f2bb348 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -131,9 +131,9 @@ # # path.data: /path/to/data # -# Can optionally include more than one location, causing data to be stripped across -# the locations (ala RAID 0) on a file level, favouring locations with most -# free space on creation. For example: +# Can optionally include more than one location, causing data to be striped across +# the locations (à la RAID 0) on a file level, favouring locations with most free +# space on creation. For example: # # path.data: /path/to/data1,/path/to/data2 From 8658856b4842131167f35a5688d84efa78dc9eb9 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Sun, 25 Sep 2011 22:20:53 +0300 Subject: [PATCH 69/96] use index iteration over iterator --- .../org/elasticsearch/cluster/routing/RoutingNodes.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index 014b2d23aeca4..4cc338c789f24 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -194,13 +194,16 @@ public List shardsRoutingFor(ShardRouting shardRouting) { public List shardsRoutingFor(String index, int shardId) { List shards = newArrayList(); for (RoutingNode routingNode : this) { - for (MutableShardRouting shardRouting : routingNode) { + List nShards = routingNode.shards(); + for (int i = 0; i < nShards.size(); i++) { + MutableShardRouting shardRouting = nShards.get(i); if (shardRouting.index().equals(index) && shardRouting.id() == shardId) { shards.add(shardRouting); } } } - for (MutableShardRouting shardRouting : unassigned) { + for (int i = 0; i < unassigned.size(); i++) { + MutableShardRouting shardRouting = unassigned.get(i); if (shardRouting.index().equals(index) && shardRouting.id() == shardId) { shards.add(shardRouting); } From e33dbcd93e3b1d0beac7e1629a790a28e5cab749 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 12:05:48 +0300 Subject: [PATCH 70/96] more small optimizations to term creation --- .../org/elasticsearch/index/mapper/FieldMapper.java | 7 +++++++ .../index/mapper/core/AbstractFieldMapper.java | 12 ++++-------- .../index/mapper/internal/AllFieldMapper.java | 2 +- .../index/mapper/internal/IndexFieldMapper.java | 2 +- .../index/mapper/internal/ParentFieldMapper.java | 2 +- .../index/mapper/internal/TypeFieldMapper.java | 4 ++-- .../index/mapper/internal/UidFieldMapper.java | 2 +- .../elasticsearch/index/mapper/ip/IpFieldMapper.java | 2 +- .../index/percolator/PercolatorService.java | 3 +-- .../index/query/PrefixFilterParser.java | 9 ++++++--- .../elasticsearch/index/query/PrefixQueryParser.java | 8 +++++--- .../elasticsearch/index/query/TermsFilterParser.java | 4 +++- .../elasticsearch/index/query/TypeFilterParser.java | 3 +-- .../org/elasticsearch/index/search/UidFilter.java | 2 +- 14 files changed, 35 insertions(+), 27 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 749dfc34e1b80..b06bdd9bc1f66 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -47,6 +47,8 @@ public static class Names { private final String fullName; + private final Term indexNameTermFactory; + public Names(String name) { this(name, name, name, name); } @@ -56,6 +58,7 @@ public Names(String name, String indexName, String indexNameClean, String fullNa this.indexName = indexName.intern(); this.indexNameClean = indexNameClean.intern(); this.fullName = fullName.intern(); + this.indexNameTermFactory = new Term(indexName, ""); } /** @@ -86,6 +89,10 @@ public String indexNameClean() { public String fullName() { return fullName; } + + public Term createIndexNameTerm(String value) { + return indexNameTermFactory.createTerm(value); + } } Names names(); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java index 23403ee5d16b3..293feb825aa90 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java @@ -216,8 +216,6 @@ protected String buildFullName(BuilderContext context) { protected final NamedAnalyzer searchAnalyzer; - protected final Term termFactory; - protected AbstractFieldMapper(Names names, Field.Index index, Field.Store store, Field.TermVector termVector, float boost, boolean omitNorms, boolean omitTermFreqAndPositions, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer) { this.names = names; @@ -238,8 +236,6 @@ protected AbstractFieldMapper(Names names, Field.Index index, Field.Store store, } else { this.searchAnalyzer = searchAnalyzer; } - - this.termFactory = new Term(names.indexName(), ""); } @Override public String name() { @@ -347,19 +343,19 @@ protected boolean customBoost() { } @Override public Query fieldQuery(String value, QueryParseContext context) { - return new TermQuery(termFactory.createTerm(indexedValue(value))); + return new TermQuery(names().createIndexNameTerm(indexedValue(value))); } @Override public Query fuzzyQuery(String value, String minSim, int prefixLength, int maxExpansions) { - return new FuzzyQuery(termFactory.createTerm(indexedValue(value)), Float.parseFloat(minSim), prefixLength, maxExpansions); + return new FuzzyQuery(names().createIndexNameTerm(indexedValue(value)), Float.parseFloat(minSim), prefixLength, maxExpansions); } @Override public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions) { - return new FuzzyQuery(termFactory.createTerm(value), (float) minSim, prefixLength, maxExpansions); + return new FuzzyQuery(names().createIndexNameTerm(value), (float) minSim, prefixLength, maxExpansions); } @Override public Filter fieldFilter(String value) { - return new TermFilter(termFactory.createTerm(indexedValue(value))); + return new TermFilter(names().createIndexNameTerm(indexedValue(value))); } @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java index 3f9b69bcb913a..cbf6ddb181b15 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java @@ -144,7 +144,7 @@ public boolean enabled() { } @Override public Query fieldQuery(String value, QueryParseContext context) { - return new AllTermQuery(termFactory.createTerm(value)); + return new AllTermQuery(names().createIndexNameTerm(value)); } @Override public void preParse(ParseContext context) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java index 78a308fd34ec1..4cb407d3f9ff5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java @@ -144,7 +144,7 @@ public String value(Document document) { } public Term term(String value) { - return termFactory.createTerm(value); + return names().createIndexNameTerm(value); } @Override public void preParse(ParseContext context) throws IOException { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index 9883a7613433a..ed56371b35059 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -179,7 +179,7 @@ public Term term(String type, String id) { } public Term term(String uid) { - return termFactory.createTerm(uid); + return names().createIndexNameTerm(uid); } @Override protected String contentType() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java index 8b6b72a7b368c..ce3eaf8b68d1b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/TypeFieldMapper.java @@ -129,14 +129,14 @@ public String value(Document document) { } public Term term(String value) { - return termFactory.createTerm(value); + return names().createIndexNameTerm(value); } @Override public Filter fieldFilter(String value) { if (index == Field.Index.NO) { return new PrefixFilter(UidFieldMapper.TERM_FACTORY.createTerm(Uid.typePrefix(value))); } - return new TermFilter(termFactory.createTerm(value)); + return new TermFilter(names().createIndexNameTerm(value)); } @Override public Query fieldQuery(String value, QueryParseContext context) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java index d96bc5934820d..48e199299ce5d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/internal/UidFieldMapper.java @@ -169,7 +169,7 @@ public Term term(String type, String id) { } public Term term(String uid) { - return termFactory.createTerm(uid); + return names().createIndexNameTerm(uid); } @Override public void close() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java index 1b26430d30e4b..be24774e7c5e1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/ip/IpFieldMapper.java @@ -195,7 +195,7 @@ protected IpFieldMapper(Names names, int precisionStep, } @Override public Query fuzzyQuery(String value, double minSim, int prefixLength, int maxExpansions) { - return new FuzzyQuery(termFactory.createTerm(value), (float) minSim, prefixLength, maxExpansions); + return new FuzzyQuery(names().createIndexNameTerm(value), (float) minSim, prefixLength, maxExpansions); } @Override public Query rangeQuery(String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java index 6ad315ee6c615..1891821e0809c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/percolator/PercolatorService.java @@ -21,7 +21,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; import org.apache.lucene.search.Collector; import org.apache.lucene.search.DeletionAwareConstantScoreQuery; import org.apache.lucene.search.Filter; @@ -138,7 +137,7 @@ private void loadQueries(String indexName) { } private Filter indexQueriesFilter(String indexName) { - return percolatorIndexService().cache().filter().cache(new TermFilter(new Term(TypeFieldMapper.NAME, indexName))); + return percolatorIndexService().cache().filter().cache(new TermFilter(TypeFieldMapper.TERM_FACTORY.createTerm(indexName))); } private boolean percolatorAllocated() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java index 693c925a287d8..260fafca44878 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixFilterParser.java @@ -77,15 +77,18 @@ public class PrefixFilterParser implements FilterParser { throw new QueryParsingException(parseContext.index(), "No value specified for prefix filter"); } + Filter filter = null; + MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { - fieldName = smartNameFieldMappers.mapper().names().indexName(); value = smartNameFieldMappers.mapper().indexedValue(value); + filter = new PrefixFilter(smartNameFieldMappers.mapper().names().createIndexNameTerm(value)); } } - - Filter filter = new PrefixFilter(new Term(fieldName, value)); + if (filter == null) { + filter = new PrefixFilter(new Term(fieldName, value)); + } if (cache) { filter = parseContext.cacheFilter(filter, cacheKey); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java index 19fd9435c9a13..f06b79c8611d8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/PrefixQueryParser.java @@ -83,15 +83,17 @@ public class PrefixQueryParser implements QueryParser { throw new QueryParsingException(parseContext.index(), "No value specified for prefix query"); } + PrefixQuery query = null; MapperService.SmartNameFieldMappers smartNameFieldMappers = parseContext.smartFieldMappers(fieldName); if (smartNameFieldMappers != null) { if (smartNameFieldMappers.hasMapper()) { - fieldName = smartNameFieldMappers.mapper().names().indexName(); value = smartNameFieldMappers.mapper().indexedValue(value); + query = new PrefixQuery(smartNameFieldMappers.mapper().names().createIndexNameTerm(value)); } } - - PrefixQuery query = new PrefixQuery(new Term(fieldName, value)); + if (query == null) { + query = new PrefixQuery(new Term(fieldName, value)); + } query.setRewriteMethod(QueryParsers.parseRewriteMethod(rewriteMethod)); query.setBoost(boost); return wrapSmartNameQuery(query, smartNameFieldMappers, parseContext); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java index d8b524b546de7..9ace0678dd25d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TermsFilterParser.java @@ -77,8 +77,10 @@ public class TermsFilterParser implements FilterParser { } if (fieldMapper != null) { value = fieldMapper.indexedValue(value); + termsFilter.addTerm(fieldMapper.names().createIndexNameTerm(value)); + } else { + termsFilter.addTerm(new Term(fieldName, value)); } - termsFilter.addTerm(new Term(fieldName, value)); } } else if (token.isValue()) { if ("_name".equals(currentFieldName)) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java index ee74547d05c1f..c1a82c8452569 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/query/TypeFilterParser.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.query; -import org.apache.lucene.index.Term; import org.apache.lucene.search.Filter; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.lucene.search.TermFilter; @@ -62,7 +61,7 @@ public class TypeFilterParser implements FilterParser { Filter filter; DocumentMapper documentMapper = parseContext.mapperService().documentMapper(type); if (documentMapper == null) { - filter = new TermFilter(new Term(TypeFieldMapper.NAME, type)); + filter = new TermFilter(TypeFieldMapper.TERM_FACTORY.createTerm(type)); } else { filter = documentMapper.typeFilter(); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java index 7d904127699e2..212cf514ffeda 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/search/UidFilter.java @@ -48,7 +48,7 @@ public UidFilter(Collection types, List ids, BloomCache bloomCac this.uids = new ArrayList(types.size() * ids.size()); for (String type : types) { for (String id : ids) { - uids.add(new Term(UidFieldMapper.NAME, Uid.createUid(type, id))); + uids.add(UidFieldMapper.TERM_FACTORY.createTerm(Uid.createUid(type, id))); } } } From 1d6fa98c2f3d7c41fd37a88064063ca4b6e34a20 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 13:57:51 +0300 Subject: [PATCH 71/96] change to elasticsearch.org site --- README.textile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.textile b/README.textile index da321d3a05ee7..d14df368b8fb0 100644 --- a/README.textile +++ b/README.textile @@ -2,7 +2,7 @@ h1. ElasticSearch h2. A Distributed RESTful Search Engine -h3. "http://www.elasticsearch.com":http://www.elasticsearch.com +h3. "http://www.elasticsearch.org":http://www.elasticsearch.org ElasticSearch is a distributed RESTful search engine built for the cloud. Features include: From 8a63e58e1a677b49297641b0ea512764adb8a409 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 21:01:23 +0300 Subject: [PATCH 72/96] Add `cloud.node.auto_attributes` setting, when set to `true`, will automatically add aws ec2 related attributes to the node (like availability zone), closes #1364. --- .../elasticsearch/cluster/ClusterModule.java | 2 + .../cluster/node/DiscoveryNode.java | 20 ----- .../cluster/node/DiscoveryNodeService.java | 85 +++++++++++++++++++ .../discovery/local/LocalDiscovery.java | 11 ++- .../discovery/zen/ZenDiscovery.java | 9 +- .../cloud/aws/AwsEc2Service.java | 7 +- .../cloud/aws/network/Ec2NameResolver.java | 5 +- .../aws/node/Ec2CustomNodeAttributes.java | 76 +++++++++++++++++ .../discovery/ec2/Ec2Discovery.java | 6 +- 9 files changed, 189 insertions(+), 32 deletions(-) create mode 100644 modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java create mode 100644 plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 7fa95f9187cbb..30c924f0b8644 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.metadata.MetaDataService; import org.elasticsearch.cluster.metadata.MetaDataStateIndexService; import org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService; +import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.routing.RoutingService; import org.elasticsearch.cluster.routing.allocation.AllocationModule; import org.elasticsearch.cluster.routing.operation.OperationRoutingModule; @@ -61,6 +62,7 @@ public ClusterModule(Settings settings) { @Override protected void configure() { + bind(DiscoveryNodeService.class).asEagerSingleton(); bind(ClusterService.class).to(InternalClusterService.class).asEagerSingleton(); bind(MetaDataService.class).asEagerSingleton(); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index 2c97b3da5826f..f741a3f11b58e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.collect.ImmutableMap; -import org.elasticsearch.common.collect.Maps; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -46,25 +45,6 @@ public static boolean nodeRequiresLocalStorage(Settings settings) { return !(settings.getAsBoolean("node.client", false) || (!settings.getAsBoolean("node.data", true) && !settings.getAsBoolean("node.master", true))); } - public static Map buildCommonNodesAttributes(Settings settings) { - Map attributes = Maps.newHashMap(settings.getByPrefix("node.").getAsMap()); - attributes.remove("name"); // name is extracted in other places - if (attributes.containsKey("client")) { - if (attributes.get("client").equals("false")) { - attributes.remove("client"); // this is the default - } else { - // if we are client node, don't store data ... - attributes.put("data", "false"); - } - } - if (attributes.containsKey("data")) { - if (attributes.get("data").equals("true")) { - attributes.remove("data"); - } - } - return attributes; - } - public static final ImmutableList EMPTY_LIST = ImmutableList.of(); private String nodeName = "".intern(); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java new file mode 100644 index 0000000000000..630b9e6548285 --- /dev/null +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNodeService.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.node; + +import org.elasticsearch.common.collect.Maps; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + */ +public class DiscoveryNodeService extends AbstractComponent { + + private final List customAttributesProviders = new CopyOnWriteArrayList(); + + @Inject public DiscoveryNodeService(Settings settings) { + super(settings); + } + + public DiscoveryNodeService addCustomAttributeProvider(CustomAttributesProvider customAttributesProvider) { + customAttributesProviders.add(customAttributesProvider); + return this; + } + + public Map buildAttributes() { + Map attributes = Maps.newHashMap(settings.getByPrefix("node.").getAsMap()); + attributes.remove("name"); // name is extracted in other places + if (attributes.containsKey("client")) { + if (attributes.get("client").equals("false")) { + attributes.remove("client"); // this is the default + } else { + // if we are client node, don't store data ... + attributes.put("data", "false"); + } + } + if (attributes.containsKey("data")) { + if (attributes.get("data").equals("true")) { + attributes.remove("data"); + } + } + + for (CustomAttributesProvider provider : customAttributesProviders) { + try { + Map customAttributes = provider.buildAttributes(); + if (customAttributes != null) { + for (Map.Entry entry : customAttributes.entrySet()) { + if (!attributes.containsKey(entry.getKey())) { + attributes.put(entry.getKey(), entry.getValue()); + } + } + } + } catch (Exception e) { + logger.warn("failed to build custom attributes from provider [{}]", e, provider); + } + } + + return attributes; + } + + public static interface CustomAttributesProvider { + + Map buildAttributes(); + } +} diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java b/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java index 1b89d03e9fbfd..a98cd7e929db8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/local/LocalDiscovery.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.ProcessedClusterStateUpdateTask; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; @@ -46,7 +47,6 @@ import java.util.concurrent.atomic.AtomicLong; import static org.elasticsearch.cluster.ClusterState.*; -import static org.elasticsearch.cluster.node.DiscoveryNode.*; import static org.elasticsearch.common.collect.Sets.*; /** @@ -58,6 +58,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem private final ClusterService clusterService; + private final DiscoveryNodeService discoveryNodeService; + private final ClusterName clusterName; private DiscoveryNode localNode; @@ -73,11 +75,13 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem private static final AtomicLong nodeIdGenerator = new AtomicLong(); - @Inject public LocalDiscovery(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService) { + @Inject public LocalDiscovery(Settings settings, ClusterName clusterName, TransportService transportService, ClusterService clusterService, + DiscoveryNodeService discoveryNodeService) { super(settings); this.clusterName = clusterName; this.clusterService = clusterService; this.transportService = transportService; + this.discoveryNodeService = discoveryNodeService; } @Override protected void doStart() throws ElasticSearchException { @@ -88,7 +92,8 @@ public class LocalDiscovery extends AbstractLifecycleComponent implem clusterGroups.put(clusterName, clusterGroup); } logger.debug("Connected to cluster [{}]", clusterName); - this.localNode = new DiscoveryNode(settings.get("name"), Long.toString(nodeIdGenerator.incrementAndGet()), transportService.boundAddress().publishAddress(), buildCommonNodesAttributes(settings)); + this.localNode = new DiscoveryNode(settings.get("name"), Long.toString(nodeIdGenerator.incrementAndGet()), transportService.boundAddress().publishAddress(), + discoveryNodeService.buildAttributes()); clusterGroup.members().add(this); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java b/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java index cafcee72d9a59..2d26b8f0c35ab 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/discovery/zen/ZenDiscovery.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.UUID; @@ -59,7 +60,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.cluster.ClusterState.*; -import static org.elasticsearch.cluster.node.DiscoveryNode.*; import static org.elasticsearch.cluster.node.DiscoveryNodes.*; import static org.elasticsearch.common.collect.Lists.*; import static org.elasticsearch.common.unit.TimeValue.*; @@ -77,6 +77,8 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen private final ClusterName clusterName; + private final DiscoveryNodeService discoveryNodeService; + private final ZenPingService pingService; private final MasterFaultDetection masterFD; @@ -110,12 +112,13 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen @Inject public ZenDiscovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, ClusterService clusterService, NodeSettingsService nodeSettingsService, - ZenPingService pingService) { + DiscoveryNodeService discoveryNodeService, ZenPingService pingService) { super(settings); this.clusterName = clusterName; this.threadPool = threadPool; this.clusterService = clusterService; this.transportService = transportService; + this.discoveryNodeService = discoveryNodeService; this.pingService = pingService; // also support direct discovery.zen settings, for cases when it gets extended @@ -138,7 +141,7 @@ public class ZenDiscovery extends AbstractLifecycleComponent implemen } @Override protected void doStart() throws ElasticSearchException { - Map nodeAttributes = buildCommonNodesAttributes(settings); + Map nodeAttributes = discoveryNodeService.buildAttributes(); // note, we rely on the fact that its a new id each time we start, see FD and "kill -9" handling String nodeId = UUID.randomBase64UUID(); localNode = new DiscoveryNode(settings.get("name"), nodeId, transportService.boundAddress().publishAddress(), nodeAttributes); diff --git a/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java index dac3fad38665f..a66261bb87a28 100644 --- a/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java +++ b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/AwsEc2Service.java @@ -27,6 +27,8 @@ import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.cloud.aws.network.Ec2NameResolver; +import org.elasticsearch.cloud.aws.node.Ec2CustomNodeAttributes; +import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.network.NetworkService; @@ -38,13 +40,16 @@ */ public class AwsEc2Service extends AbstractLifecycleComponent { + public static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; + private AmazonEC2Client client; - @Inject public AwsEc2Service(Settings settings, SettingsFilter settingsFilter, NetworkService networkService) { + @Inject public AwsEc2Service(Settings settings, SettingsFilter settingsFilter, NetworkService networkService, DiscoveryNodeService discoveryNodeService) { super(settings); settingsFilter.addFilter(new AwsSettingsFilter()); // add specific ec2 name resolver networkService.addCustomNameResolver(new Ec2NameResolver(settings)); + discoveryNodeService.addCustomAttributeProvider(new Ec2CustomNodeAttributes(settings)); } public synchronized AmazonEC2 client() { diff --git a/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java index b9785394935e3..6534fc247e34f 100755 --- a/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java +++ b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/network/Ec2NameResolver.java @@ -20,6 +20,7 @@ package org.elasticsearch.cloud.aws.network; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.io.Closeables; import org.elasticsearch.common.network.NetworkService.CustomNameResolver; @@ -78,8 +79,6 @@ private Ec2HostnameType(String configName, String ec2Name) { } } - private static final String EC2_METADATA_URL = "http://169.254.169.254/latest/meta-data/"; - /** * Construct a {@link CustomNameResolver}. */ @@ -97,7 +96,7 @@ public InetAddress resolve(Ec2HostnameType type, boolean warnOnFailure) { URLConnection urlConnection = null; InputStream in = null; try { - URL url = new URL(EC2_METADATA_URL + type.ec2Name); + URL url = new URL(AwsEc2Service.EC2_METADATA_URL + type.ec2Name); logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url); urlConnection = url.openConnection(); urlConnection.setConnectTimeout(2000); diff --git a/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java new file mode 100644 index 0000000000000..dd942b1a9af7b --- /dev/null +++ b/plugins/cloud/aws/src/main/java/org/elasticsearch/cloud/aws/node/Ec2CustomNodeAttributes.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cloud.aws.node; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cloud.aws.AwsEc2Service; +import org.elasticsearch.cluster.node.DiscoveryNodeService; +import org.elasticsearch.common.collect.Maps; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.io.Closeables; +import org.elasticsearch.common.settings.Settings; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; +import java.util.Map; + +/** + */ +public class Ec2CustomNodeAttributes extends AbstractComponent implements DiscoveryNodeService.CustomAttributesProvider { + + public Ec2CustomNodeAttributes(Settings settings) { + super(settings); + } + + @Override public Map buildAttributes() { + if (!settings.getAsBoolean("cloud.node.auto_attributes", false)) { + return null; + } + Map ec2Attributes = Maps.newHashMap(); + + URLConnection urlConnection; + InputStream in = null; + try { + URL url = new URL(AwsEc2Service.EC2_METADATA_URL + "placement/availability-zone"); + logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url); + urlConnection = url.openConnection(); + urlConnection.setConnectTimeout(2000); + in = urlConnection.getInputStream(); + BufferedReader urlReader = new BufferedReader(new InputStreamReader(in)); + + String metadataResult = urlReader.readLine(); + if (metadataResult == null || metadataResult.length() == 0) { + logger.error("no ec2 metadata returned from {}", url); + return null; + } + ec2Attributes.put("aws_availability_zone", metadataResult); + } catch (IOException e) { + logger.debug("failed to get metadata for [placement/availability-zone]: " + ExceptionsHelper.detailedMessage(e)); + } finally { + Closeables.closeQuietly(in); + } + + return ec2Attributes; + } +} diff --git a/plugins/cloud/aws/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java b/plugins/cloud/aws/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java index 26144d10c0786..7005860717ffe 100755 --- a/plugins/cloud/aws/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java +++ b/plugins/cloud/aws/src/main/java/org/elasticsearch/discovery/ec2/Ec2Discovery.java @@ -22,6 +22,7 @@ import org.elasticsearch.cloud.aws.AwsEc2Service; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.cluster.node.DiscoveryNodeService; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -39,8 +40,9 @@ public class Ec2Discovery extends ZenDiscovery { @Inject public Ec2Discovery(Settings settings, ClusterName clusterName, ThreadPool threadPool, TransportService transportService, - ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, AwsEc2Service ec2Service) { - super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, pingService); + ClusterService clusterService, NodeSettingsService nodeSettingsService, ZenPingService pingService, + DiscoveryNodeService discoveryNodeService, AwsEc2Service ec2Service) { + super(settings, clusterName, threadPool, transportService, clusterService, nodeSettingsService, discoveryNodeService, pingService); if (settings.getAsBoolean("cloud.enabled", true)) { ImmutableList zenPings = pingService.zenPings(); UnicastZenPing unicastZenPing = null; From 79b63b3e07f3b79e02d9cdb0cbf94acc21592e30 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 21:20:25 +0300 Subject: [PATCH 73/96] add junit to the test dep when running groovy tests --- plugins/lang/groovy/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/lang/groovy/build.gradle b/plugins/lang/groovy/build.gradle index 0b53edd4ced58..b1f9098dbe5bd 100644 --- a/plugins/lang/groovy/build.gradle +++ b/plugins/lang/groovy/build.gradle @@ -32,7 +32,7 @@ dependencies { groovy group: 'org.codehaus.groovy', name: 'groovy-all', version: '1.8.2' distLib('org.codehaus.groovy:groovy-all:1.8.2') { transitive = false } - //testCompile('junit:junit:4.8.1') {transitive = false} + testCompile('junit:junit:4.8.1') {transitive = false} } task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << { From eca11d1e610e65a86dc28a69016d43e94aeb776e Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 21:30:18 +0300 Subject: [PATCH 74/96] add not attributes doc, and env var automatic replacement --- config/elasticsearch.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 666962f2bb348..5f12e74e36399 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -15,6 +15,11 @@ # effect of certain configuration option, please _do ask_ on the # mailing list or IRC channel [http://elasticsearch.org/community]. +# Any element in the configuration can be replaced with environment variables +# by placing them in ${...} notation. For example: +# +# node.rack: ${RACK_ENV_VAR} + # See # for information on supported formats and syntax for the configuration file. @@ -71,6 +76,12 @@ # such as and # to inspect the cluster state. +# A node can have generic attributes associated with it, which can later be used +# for customized shard allocation filtering, or allocation awareness. An attribute +# is a simple key value pair, similar to node.key: value, here is an example: +# +# node.rack: rack314 + #################################### Index #################################### From c5835cac04051736726f5835769cfc9b9003eb7f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 26 Sep 2011 21:32:46 +0300 Subject: [PATCH 75/96] more doc in config --- config/elasticsearch.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/config/elasticsearch.yml b/config/elasticsearch.yml index 5f12e74e36399..496938eddb9bb 100644 --- a/config/elasticsearch.yml +++ b/config/elasticsearch.yml @@ -181,7 +181,9 @@ ############################## Network And HTTP ############################### # ElasticSearch, by default, binds itself to the 0.0.0.0 address, and listens -# on port 9200 for HTTP traffic and on port 9300 for node-to-node communication. +# on port [9200-9300] for HTTP traffic and on port [9300-9400] for node-to-node +# communication. (the range means that if the port is busy, it will automatically +# try the next port). # Set the bind address specifically (IPv4 or IPv6): # @@ -200,7 +202,7 @@ # # transport.port: 9300 -# Enable compression for all communication between (disabled by default): +# Enable compression for all communication between nodes (disabled by default): # # transport.tcp.compress: true From d611182dbfc58d35938d95c28527ae2c264b3880 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 26 Sep 2011 15:30:06 -0400 Subject: [PATCH 76/96] Add support for non-elasticsearch namespaces to ImmutableSettings.getAsClass method --- .../common/settings/ImmutableSettings.java | 12 +++- .../settings/ImmutableSettingsTests.java | 62 +++++++++++++++++++ .../common/settings/bar/BarTest.java | 26 ++++++++ .../common/settings/foo/FooTest.java | 26 ++++++++ 4 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/ImmutableSettingsTests.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/bar/BarTest.java create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/foo/FooTest.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java index e917cf8437b63..20c4f62d81e14 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/settings/ImmutableSettings.java @@ -213,15 +213,21 @@ private ImmutableSettings(Map settings, ClassLoader classLoader) try { return (Class) getClassLoader().loadClass(fullClassName); } catch (ClassNotFoundException e) { - fullClassName = prefixPackage + Strings.capitalize(toCamelCase(sValue)) + suffixClassName; + String prefixValue = prefixPackage; + int packageSeparator = sValue.lastIndexOf('.'); + if (packageSeparator > 0) { + prefixValue = sValue.substring(0, packageSeparator + 1); + sValue = sValue.substring(packageSeparator + 1); + } + fullClassName = prefixValue + Strings.capitalize(toCamelCase(sValue)) + suffixClassName; try { return (Class) getClassLoader().loadClass(fullClassName); } catch (ClassNotFoundException e1) { - fullClassName = prefixPackage + toCamelCase(sValue).toLowerCase() + "." + Strings.capitalize(toCamelCase(sValue)) + suffixClassName; + fullClassName = prefixValue + toCamelCase(sValue).toLowerCase() + "." + Strings.capitalize(toCamelCase(sValue)) + suffixClassName; try { return (Class) getClassLoader().loadClass(fullClassName); } catch (ClassNotFoundException e2) { - throw new NoClassSettingsException("Failed to load class setting [" + setting + "] with value [" + sValue + "]", e); + throw new NoClassSettingsException("Failed to load class setting [" + setting + "] with value [" + get(setting) + "]", e); } } } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/ImmutableSettingsTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/ImmutableSettingsTests.java new file mode 100644 index 0000000000000..babf79c114a5f --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/ImmutableSettingsTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.common.settings.foo.FooTest; +import org.testng.annotations.Test; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + * @author imotov + */ +public class ImmutableSettingsTests { + + @Test public void testGetAsClass() { + Settings settings = settingsBuilder() + .put("test.class", "bar") + .put("test.class.package", "org.elasticsearch.common.settings.bar") + .build(); + + // Assert that defaultClazz is loaded if setting is not specified + assertThat(settings.getAsClass("no.settings", FooTest.class, "org.elasticsearch.common.settings.", "Test").getName(), + equalTo("org.elasticsearch.common.settings.foo.FooTest")); + + // Assert that correct class is loaded if setting contain name without package + assertThat(settings.getAsClass("test.class", FooTest.class, "org.elasticsearch.common.settings.", "Test").getName(), + equalTo("org.elasticsearch.common.settings.bar.BarTest")); + + // Assert that class cannot be loaded if wrong packagePrefix is specified + try { + settings.getAsClass("test.class", FooTest.class, "com.example.elasticsearch.common.settings.", "Test"); + assertThat("Class with wrong package name shouldn't be loaded", false); + } catch (NoClassSettingsException ex) { + // Ignore + } + + // Assert that package name in settings is getting correctly applied + assertThat(settings.getAsClass("test.class.package", FooTest.class, "com.example.elasticsearch.common.settings.", "Test").getName(), + equalTo("org.elasticsearch.common.settings.bar.BarTest")); + + } + +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/bar/BarTest.java b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/bar/BarTest.java new file mode 100644 index 0000000000000..c578e1bf7cfac --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/bar/BarTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings.bar; + +/** + * @author imotov + */ +public class BarTest { +} diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/foo/FooTest.java b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/foo/FooTest.java new file mode 100644 index 0000000000000..0b44bec1b606e --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/common/settings/foo/FooTest.java @@ -0,0 +1,26 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings.foo; + +/** + * @author imotov + */ +public class FooTest { +} From 737589f50d85f631fec2c59c86e8d175eb639211 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 00:39:37 +0300 Subject: [PATCH 77/96] better handling of mean computation of some facets to handle division by 0, though, won't happen on most facets --- .../datehistogram/InternalFullDateHistogramFacet.java | 11 ++++++++++- .../search/facet/geodistance/GeoDistanceFacet.java | 3 +++ .../bounded/InternalBoundedFullHistogramFacet.java | 3 +++ .../elasticsearch/search/facet/range/RangeFacet.java | 3 +++ .../facet/statistical/InternalStatisticalFacet.java | 3 +++ .../doubles/InternalTermsStatsDoubleFacet.java | 11 ++++++++++- .../termsstats/longs/InternalTermsStatsLongFacet.java | 11 ++++++++++- .../strings/InternalTermsStatsStringFacet.java | 11 ++++++++++- 8 files changed, 52 insertions(+), 4 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java index a915b8d5917b5..4e54d21276240 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalFullDateHistogramFacet.java @@ -28,7 +28,13 @@ import org.elasticsearch.search.facet.Facet; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; /** * @author kimchy (shay.banon) @@ -105,6 +111,9 @@ public FullEntry(long time, long count, double min, double max, long totalCount, } @Override public double mean() { + if (totalCount == 0) { + return totalCount; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacet.java index dc025cade6235..f724508581434 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/geodistance/GeoDistanceFacet.java @@ -118,6 +118,9 @@ public double getTotal() { * The mean of this facet interval. */ public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/bounded/InternalBoundedFullHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/bounded/InternalBoundedFullHistogramFacet.java index 34b0c8197b304..b63794e534c53 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/bounded/InternalBoundedFullHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/bounded/InternalBoundedFullHistogramFacet.java @@ -109,6 +109,9 @@ public FullEntry(long key, long count, double min, double max, long totalCount, } @Override public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacet.java index 13604f9582b96..0fd39cdf3043d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/RangeFacet.java @@ -134,6 +134,9 @@ public double getTotal() { * The mean of this facet interval. */ public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java index 6032f159b66b8..7c273b587fc58 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/statistical/InternalStatisticalFacet.java @@ -114,6 +114,9 @@ public InternalStatisticalFacet(String name, double min, double max, double tota } @Override public double mean() { + if (count == 0) { + return 0; + } return total / count; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java index 7f4855cee3bcd..ec62677352266 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/doubles/InternalTermsStatsDoubleFacet.java @@ -30,7 +30,13 @@ import org.elasticsearch.search.facet.termsstats.InternalTermsStatsFacet; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; public class InternalTermsStatsDoubleFacet extends InternalTermsStatsFacet { @@ -128,6 +134,9 @@ public DoubleEntry(double term, long count, long totalCount, double total, doubl } @Override public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java index a54ace7635892..12d245109dbe6 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/longs/InternalTermsStatsLongFacet.java @@ -30,7 +30,13 @@ import org.elasticsearch.search.facet.termsstats.InternalTermsStatsFacet; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; public class InternalTermsStatsLongFacet extends InternalTermsStatsFacet { @@ -127,6 +133,9 @@ public LongEntry(long term, long count, long totalCount, double total, double mi } @Override public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java index 4f6ab4ef1c3fe..262fb2b9398f9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/termsstats/strings/InternalTermsStatsStringFacet.java @@ -30,7 +30,13 @@ import org.elasticsearch.search.facet.termsstats.InternalTermsStatsFacet; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; public class InternalTermsStatsStringFacet extends InternalTermsStatsFacet { @@ -128,6 +134,9 @@ public StringEntry(String term, long count, long totalCount, double total, doubl } @Override public double mean() { + if (totalCount == 0) { + return 0; + } return total / totalCount; } From 0c82fc5901c4ca81ff9226f703cf575194c92948 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 00:41:57 +0300 Subject: [PATCH 78/96] Remove Infinity values for Range facets when no docs match the range, closes #1366. --- .../search/facet/range/InternalRangeFacet.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java index 47d55af929c9e..722de608992ee 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/range/InternalRangeFacet.java @@ -180,8 +180,11 @@ static final class Fields { builder.field(Fields.TO_STR, entry.toAsString); } builder.field(Fields.COUNT, entry.count()); - builder.field(Fields.MIN, entry.min()); - builder.field(Fields.MAX, entry.max()); + // only output min and max if there are actually documents matching this range... + if (entry.totalCount() > 0) { + builder.field(Fields.MIN, entry.min()); + builder.field(Fields.MAX, entry.max()); + } builder.field(Fields.TOTAL_COUNT, entry.totalCount()); builder.field(Fields.TOTAL, entry.total()); builder.field(Fields.MEAN, entry.mean()); From e0fdccd9c017440cb1b8c813497ee7996cfb823a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 01:29:03 +0300 Subject: [PATCH 79/96] Transport Client: When `sniff` is enabled, use the sniffed nodes to be the list fo nodes to ping as well as the provided nodes, closes #1217. --- .../TransportClientNodesService.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java index a5f308b60e575..7cdfc2d78103a 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.collect.ImmutableList; +import org.elasticsearch.common.collect.Maps; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -43,6 +44,7 @@ import java.util.HashSet; import java.util.Iterator; +import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; @@ -288,11 +290,21 @@ class SniffNodesSampler implements NodeSampler { if (closed) { return; } - ImmutableList listedNodes = TransportClientNodesService.this.listedNodes; - final CountDownLatch latch = new CountDownLatch(listedNodes.size()); + + // the nodes we are going to ping include the core listed nodes that were added + // and the last round of discovered nodes + Map nodesToPing = Maps.newHashMap(); + for (DiscoveryNode node : listedNodes) { + nodesToPing.put(node.address(), node); + } + for (DiscoveryNode node : nodes) { + nodesToPing.put(node.address(), node); + } + + final CountDownLatch latch = new CountDownLatch(nodesToPing.size()); final CopyOnWriteArrayList nodesInfoResponses = new CopyOnWriteArrayList(); - for (final DiscoveryNode listedNode : listedNodes) { - threadPool.cached().execute(new Runnable() { + for (final DiscoveryNode listedNode : nodesToPing.values()) { + threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new Runnable() { @Override public void run() { try { transportService.connectToNode(listedNode); // make sure we are connected to it From 8fd28320e4589dea2bab316844b06cb7fff36973 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 11:54:34 +0300 Subject: [PATCH 80/96] Add an option to disallow deleting all indices, closes #1367. --- .../admin/indices/delete/TransportDeleteIndexAction.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java index af4017800030d..ec85d2ee07381 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.delete; import org.elasticsearch.ElasticSearchException; +import org.elasticsearch.ElasticSearchIllegalArgumentException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.TransportActions; import org.elasticsearch.action.admin.indices.mapping.delete.DeleteMappingRequest; @@ -52,11 +53,15 @@ public class TransportDeleteIndexAction extends TransportMasterNodeOperationActi private final TransportDeleteMappingAction deleteMappingAction; + private final boolean disableDeleteAllIndices; + @Inject public TransportDeleteIndexAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, MetaDataDeleteIndexService deleteIndexService, TransportDeleteMappingAction deleteMappingAction) { super(settings, transportService, clusterService, threadPool); this.deleteIndexService = deleteIndexService; this.deleteMappingAction = deleteMappingAction; + + this.disableDeleteAllIndices = settings.getAsBoolean("action.disable_delete_all_indices", false); } @Override protected String executor() { @@ -76,6 +81,9 @@ public class TransportDeleteIndexAction extends TransportMasterNodeOperationActi } @Override protected void doExecute(DeleteIndexRequest request, ActionListener listener) { + if (disableDeleteAllIndices && (request.indices() == null || request.indices().length == 0)) { + throw new ElasticSearchIllegalArgumentException("deleting all indices is disabled"); + } request.indices(clusterService.state().metaData().concreteIndices(request.indices())); super.doExecute(request, listener); } From f63727e3c6c16b995099ff5a9c1f29635a23c7ed Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 13:21:20 +0300 Subject: [PATCH 81/96] no need to log a failure when deleting an index and a shard is recovering --- .../index/engine/robin/RobinEngine.java | 15 ++- .../stress/refresh/RefreshStressTest1.java | 92 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index 064657cd1e097..1e6c6e71716f4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -125,8 +125,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { private final boolean asyncLoadBloomFilter; - // no need for volatile, its always used under a lock - private IndexWriter indexWriter; + private volatile IndexWriter indexWriter; private volatile AcquirableResource nrtResource; @@ -238,6 +237,9 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine { if (indexWriter != null) { throw new EngineAlreadyStartedException(shardId); } + if (closed) { + throw new EngineClosedException(shardId); + } if (logger.isDebugEnabled()) { logger.debug("Starting engine"); } @@ -1033,6 +1035,9 @@ private void refreshVersioningTable(long time) { } catch (Exception e) { --disableFlushCounter; phase1Snapshot.release(); + if (closed) { + e = new EngineClosedException(shardId, e); + } throw new RecoveryEngineException(shardId, 1, "Execution failed", e); } @@ -1042,6 +1047,9 @@ private void refreshVersioningTable(long time) { } catch (Exception e) { --disableFlushCounter; phase1Snapshot.release(); + if (closed) { + e = new EngineClosedException(shardId, e); + } throw new RecoveryEngineException(shardId, 2, "Snapshot failed", e); } @@ -1051,6 +1059,9 @@ private void refreshVersioningTable(long time) { --disableFlushCounter; phase1Snapshot.release(); phase2Snapshot.release(); + if (closed) { + e = new EngineClosedException(shardId, e); + } throw new RecoveryEngineException(shardId, 2, "Execution failed", e); } diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java new file mode 100644 index 0000000000000..712b5d4b5822a --- /dev/null +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java @@ -0,0 +1,92 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.stress.refresh; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.index.query.FilterBuilders; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeBuilder; + +import java.util.UUID; + +/** + */ +public class RefreshStressTest1 { + + public static void main(String[] args) throws InterruptedException { + Node node = NodeBuilder.nodeBuilder().local(true).loadConfigSettings(false).clusterName("testCluster").settings( + ImmutableSettings.settingsBuilder() + .put("node.name", "node1") + .put("gateway.type", "none") + //.put("path.data", new File("target/data").getAbsolutePath()) + .build()).node(); + Node node2 = NodeBuilder.nodeBuilder().local(true).loadConfigSettings(false).clusterName("testCluster").settings( + ImmutableSettings.settingsBuilder() + .put("node.name", "node2") + .put("gateway.type", "none") + //.put("path.data", new File("target/data").getAbsolutePath()) + .build()).node(); + Client client = node.client(); + + for (int loop = 1; loop < 1000; loop++) { + String indexName = "testindex" + loop; + String typeName = "testType" + loop; + String id = UUID.randomUUID().toString(); + String mapping = "{ \"" + typeName + "\" : {\"dynamic_templates\" : [{\"no_analyze_strings\" : {\"match_mapping_type\" : \"string\",\"match\" : \"*\",\"mapping\" : {\"type\" : \"string\",\"index\" : \"not_analyzed\"}}}]}}"; + client.admin().indices().prepareCreate(indexName).execute().actionGet(); + client.admin().indices().preparePutMapping(indexName).setType(typeName).setSource(mapping).execute().actionGet(); +// sleep after put mapping +// Thread.sleep(100); + + System.out.println("indexing " + loop); + String name = "name" + id; + client.prepareIndex(indexName, typeName, id).setSource("{ \"id\": \"" + id + "\", \"name\": \"" + name + "\" }").execute().actionGet(); + + client.admin().indices().prepareRefresh(indexName).execute().actionGet(); +// sleep after refresh +// Thread.sleep(100); + + System.out.println("searching " + loop); + SearchResponse result = client.prepareSearch(indexName).setFilter(FilterBuilders.termFilter("name", name)).execute().actionGet(); + if (result.getHits().hits().length != 1) { + for (int i = 1; i <= 100; i++) { + System.out.println("retry " + loop + ", " + i); + client.admin().indices().prepareRefresh(indexName).execute().actionGet(); + Thread.sleep(100); + result = client.prepareSearch(indexName).setFilter(FilterBuilders.termFilter("name", name)).execute().actionGet(); + if (result.getHits().hits().length == 1) { + throw new RuntimeException("Record found after " + (i * 100) + " ms"); + } else if (i == 100) { + if (client.prepareGet(indexName, typeName, id).execute().actionGet().isExists()) + throw new RuntimeException("Record wasn't found after 10s but can be get by id"); + else throw new RuntimeException("Record wasn't found after 10s and can't be get by id"); + } + } + } + + client.admin().indices().prepareDelete(indexName).execute().actionGet(); + } + client.close(); + node2.close(); + node.close(); + } +} From 4088236cf75b793c7a2a80f5f7be5b326b607378 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 13:47:59 +0300 Subject: [PATCH 82/96] No need to reroute (check for possible shard allocations) when a new *non* data node is added to the cluster, closes #1368. --- .../org/elasticsearch/cluster/routing/RoutingService.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java index f16d9c4ed2453..951c6ffde22a7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/routing/RoutingService.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.common.component.AbstractLifecycleComponent; @@ -103,7 +104,12 @@ public class RoutingService extends AbstractLifecycleComponent i // reroute(); } else { if (event.nodesAdded()) { - routingTableDirty = true; + for (DiscoveryNode node : event.nodesDelta().addedNodes()) { + if (node.dataNode()) { + routingTableDirty = true; + break; + } + } } } } else { From de8644d95a216c0251ca483f3c7fdcffc7d94547 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 21:45:10 +0300 Subject: [PATCH 83/96] Fetch phase when searching might fail when mapping are updated with type missing, closes #1369. --- .../index/mapper/MapperService.java | 84 +++++++++++-------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 753d8e648028d..e66f5462e25be 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -167,6 +167,8 @@ public void add(String type, String mappingSource) { } } + // never expose this to the outside world, we need to reparse the doc mapper so we get fresh + // instances of field mappers to properly remove existing doc mapper private void add(DocumentMapper mapper) { synchronized (mutex) { if (mapper.type().charAt(0) == '_') { @@ -181,10 +183,17 @@ private void add(DocumentMapper mapper) { if (mapper.type().contains(".")) { logger.warn("Type [{}] contains a '.', it is recommended not to include it within a type name", mapper.type()); } - remove(mapper.type()); // first remove it (in case its an update, we need to remove the aggregated mappers) + // we can add new field/object mappers while the old ones are there + // since we get new instances of those, and when we remove, we remove + // by instance equality + DocumentMapper oldMapper = mappers.get(mapper.type()); mapper.addFieldMapperListener(fieldMapperListener, true); mapper.addObjectMapperListener(objectMapperListener, true); mappers = newMapBuilder(mappers).put(mapper.type(), mapper).immutableMap(); + if (oldMapper != null) { + removeObjectFieldMappers(oldMapper); + oldMapper.close(); + } } } @@ -196,49 +205,52 @@ public void remove(String type) { } docMapper.close(); mappers = newMapBuilder(mappers).remove(type).immutableMap(); + removeObjectFieldMappers(docMapper); + } + } - // we need to remove those mappers - for (FieldMapper mapper : docMapper.mappers()) { - FieldMappers mappers = nameFieldMappers.get(mapper.names().name()); - if (mappers != null) { - mappers = mappers.remove(mapper); - if (mappers.isEmpty()) { - nameFieldMappers = newMapBuilder(nameFieldMappers).remove(mapper.names().name()).immutableMap(); - } else { - nameFieldMappers = newMapBuilder(nameFieldMappers).put(mapper.names().name(), mappers).immutableMap(); - } + private void removeObjectFieldMappers(DocumentMapper docMapper) { + // we need to remove those mappers + for (FieldMapper mapper : docMapper.mappers()) { + FieldMappers mappers = nameFieldMappers.get(mapper.names().name()); + if (mappers != null) { + mappers = mappers.remove(mapper); + if (mappers.isEmpty()) { + nameFieldMappers = newMapBuilder(nameFieldMappers).remove(mapper.names().name()).immutableMap(); + } else { + nameFieldMappers = newMapBuilder(nameFieldMappers).put(mapper.names().name(), mappers).immutableMap(); } + } - mappers = indexNameFieldMappers.get(mapper.names().indexName()); - if (mappers != null) { - mappers = mappers.remove(mapper); - if (mappers.isEmpty()) { - indexNameFieldMappers = newMapBuilder(indexNameFieldMappers).remove(mapper.names().indexName()).immutableMap(); - } else { - indexNameFieldMappers = newMapBuilder(indexNameFieldMappers).put(mapper.names().indexName(), mappers).immutableMap(); - } + mappers = indexNameFieldMappers.get(mapper.names().indexName()); + if (mappers != null) { + mappers = mappers.remove(mapper); + if (mappers.isEmpty()) { + indexNameFieldMappers = newMapBuilder(indexNameFieldMappers).remove(mapper.names().indexName()).immutableMap(); + } else { + indexNameFieldMappers = newMapBuilder(indexNameFieldMappers).put(mapper.names().indexName(), mappers).immutableMap(); } + } - mappers = fullNameFieldMappers.get(mapper.names().fullName()); - if (mappers != null) { - mappers = mappers.remove(mapper); - if (mappers.isEmpty()) { - fullNameFieldMappers = newMapBuilder(fullNameFieldMappers).remove(mapper.names().fullName()).immutableMap(); - } else { - fullNameFieldMappers = newMapBuilder(fullNameFieldMappers).put(mapper.names().fullName(), mappers).immutableMap(); - } + mappers = fullNameFieldMappers.get(mapper.names().fullName()); + if (mappers != null) { + mappers = mappers.remove(mapper); + if (mappers.isEmpty()) { + fullNameFieldMappers = newMapBuilder(fullNameFieldMappers).remove(mapper.names().fullName()).immutableMap(); + } else { + fullNameFieldMappers = newMapBuilder(fullNameFieldMappers).put(mapper.names().fullName(), mappers).immutableMap(); } } + } - for (ObjectMapper mapper : docMapper.objectMappers().values()) { - ObjectMappers mappers = objectMappers.get(mapper.fullPath()); - if (mappers != null) { - mappers = mappers.remove(mapper); - if (mappers.isEmpty()) { - objectMappers = newMapBuilder(objectMappers).remove(mapper.fullPath()).immutableMap(); - } else { - objectMappers = newMapBuilder(objectMappers).put(mapper.fullPath(), mappers).immutableMap(); - } + for (ObjectMapper mapper : docMapper.objectMappers().values()) { + ObjectMappers mappers = objectMappers.get(mapper.fullPath()); + if (mappers != null) { + mappers = mappers.remove(mapper); + if (mappers.isEmpty()) { + objectMappers = newMapBuilder(objectMappers).remove(mapper.fullPath()).immutableMap(); + } else { + objectMappers = newMapBuilder(objectMappers).put(mapper.fullPath(), mappers).immutableMap(); } } } From df3fa9c067f2accdf39e0add07900eace19f9d07 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 21:46:24 +0300 Subject: [PATCH 84/96] When refreshing, also execute the refresh operation on initializing shards to make sure we don't miss it, closes #1370. --- .../action/admin/indices/refresh/TransportRefreshAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java index f8a5852306b35..aa27b45859e78 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/refresh/TransportRefreshAction.java @@ -120,6 +120,6 @@ public class TransportRefreshAction extends TransportBroadcastOperationAction Date: Tue, 27 Sep 2011 22:56:27 +0300 Subject: [PATCH 85/96] execute the indices store listener as a last listener, to make sure we clean things up before we delete content if needed --- .../java/org/elasticsearch/cluster/ClusterService.java | 5 +++++ .../cluster/service/InternalClusterService.java | 8 ++++++++ .../org/elasticsearch/indices/store/IndicesStore.java | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterService.java index e2fff7d587245..ea15ff2112ea4 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/ClusterService.java @@ -59,6 +59,11 @@ public interface ClusterService extends LifecycleComponent { */ void addPriority(ClusterStateListener listener); + /** + * Adds last listener. + */ + void addLast(ClusterStateListener listener); + /** * Adds a listener for updated cluster states. */ diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java index 3c6708cae6bdc..16bcb84d4ed2f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/service/InternalClusterService.java @@ -79,6 +79,7 @@ public class InternalClusterService extends AbstractLifecycleComponent priorityClusterStateListeners = new CopyOnWriteArrayList(); private final List clusterStateListeners = new CopyOnWriteArrayList(); + private final List lastClusterStateListeners = new CopyOnWriteArrayList(); private final Queue onGoingTimeouts = new LinkedTransferQueue(); @@ -152,6 +153,10 @@ public void addPriority(ClusterStateListener listener) { priorityClusterStateListeners.add(listener); } + public void addLast(ClusterStateListener listener) { + lastClusterStateListeners.add(listener); + } + public void add(ClusterStateListener listener) { clusterStateListeners.add(listener); } @@ -264,6 +269,9 @@ public void submitStateUpdateTask(final String source, final ClusterStateUpdateT for (ClusterStateListener listener : clusterStateListeners) { listener.clusterChanged(clusterChangedEvent); } + for (ClusterStateListener listener : lastClusterStateListeners) { + listener.clusterChanged(clusterChangedEvent); + } if (!nodesDelta.removedNodes().isEmpty()) { threadPool.cached().execute(new Runnable() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index 24dfa3456c9bc..0a49d51869188 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -54,7 +54,7 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe this.nodeEnv = nodeEnv; this.indicesService = indicesService; this.clusterService = clusterService; - clusterService.add(this); + clusterService.addLast(this); } public void close() { From c47f73a1ea789d30b669247f6e7c1b4b7fbdd4e4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Tue, 27 Sep 2011 22:56:44 +0300 Subject: [PATCH 86/96] better refresh stress test --- .../test/stress/refresh/RefreshStressTest1.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java index 712b5d4b5822a..3801f681d7e87 100644 --- a/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/stress/refresh/RefreshStressTest1.java @@ -33,16 +33,19 @@ public class RefreshStressTest1 { public static void main(String[] args) throws InterruptedException { + int numberOfShards = 5; Node node = NodeBuilder.nodeBuilder().local(true).loadConfigSettings(false).clusterName("testCluster").settings( ImmutableSettings.settingsBuilder() .put("node.name", "node1") .put("gateway.type", "none") + .put("index.number_of_shards", numberOfShards) //.put("path.data", new File("target/data").getAbsolutePath()) .build()).node(); Node node2 = NodeBuilder.nodeBuilder().local(true).loadConfigSettings(false).clusterName("testCluster").settings( ImmutableSettings.settingsBuilder() .put("node.name", "node2") .put("gateway.type", "none") + .put("index.number_of_shards", numberOfShards) //.put("path.data", new File("target/data").getAbsolutePath()) .build()).node(); Client client = node.client(); @@ -69,12 +72,14 @@ public static void main(String[] args) throws InterruptedException { SearchResponse result = client.prepareSearch(indexName).setFilter(FilterBuilders.termFilter("name", name)).execute().actionGet(); if (result.getHits().hits().length != 1) { for (int i = 1; i <= 100; i++) { - System.out.println("retry " + loop + ", " + i); + System.out.println("retry " + loop + ", " + i + ", previous total hits: " + result.getHits().getTotalHits()); client.admin().indices().prepareRefresh(indexName).execute().actionGet(); Thread.sleep(100); result = client.prepareSearch(indexName).setFilter(FilterBuilders.termFilter("name", name)).execute().actionGet(); if (result.getHits().hits().length == 1) { - throw new RuntimeException("Record found after " + (i * 100) + " ms"); + client.admin().indices().prepareRefresh(indexName).execute().actionGet(); + result = client.prepareSearch(indexName).setFilter(FilterBuilders.termFilter("name", name)).execute().actionGet(); + throw new RuntimeException("Record found after " + (i * 100) + " ms, second go: " + result.getHits().hits().length); } else if (i == 100) { if (client.prepareGet(indexName, typeName, id).execute().actionGet().isExists()) throw new RuntimeException("Record wasn't found after 10s but can be get by id"); @@ -83,7 +88,7 @@ public static void main(String[] args) throws InterruptedException { } } - client.admin().indices().prepareDelete(indexName).execute().actionGet(); + //client.admin().indices().prepareDelete(indexName).execute().actionGet(); } client.close(); node2.close(); From d7f7f77d81da9a3d55d31b1bd1716ef82da5894e Mon Sep 17 00:00:00 2001 From: Nicolas Huray Date: Wed, 10 Aug 2011 11:04:29 +0200 Subject: [PATCH 87/96] Build ElasticSearch as Debian package --- .gitignore | 7 ++ build.gradle | 28 ++++- pkg/debian/control/conffiles | 4 + pkg/debian/control/control | 36 ++++++ pkg/debian/control/postinst | 39 +++++++ pkg/debian/control/postrm | 30 +++++ pkg/debian/control/prerm | 10 ++ pkg/debian/default/elasticsearch | 24 ++++ pkg/debian/init.d/elasticsearch | 192 +++++++++++++++++++++++++++++++ 9 files changed, 369 insertions(+), 1 deletion(-) create mode 100644 pkg/debian/control/conffiles create mode 100644 pkg/debian/control/control create mode 100755 pkg/debian/control/postinst create mode 100755 pkg/debian/control/postrm create mode 100755 pkg/debian/control/prerm create mode 100644 pkg/debian/default/elasticsearch create mode 100644 pkg/debian/init.d/elasticsearch diff --git a/.gitignore b/.gitignore index 39b25c201230d..22e2e5898af3c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,13 @@ plugins/river/twitter/build plugins/river/wikipedia/build plugins/transport/memcached/build plugins/transport/thrift/build +.idea/gradle.xml +.idea/misc.xml +.idea/modules/elasticsearch-root.iml +.idea/modules/plugin-river-rabbitmq.iml +.idea/projectCodeStyle.xml + + ## eclipse ignores (use 'gradle eclipse' to build eclipse projects) .project diff --git a/build.gradle b/build.gradle index a63f953e30193..695ff03658b40 100644 --- a/build.gradle +++ b/build.gradle @@ -94,6 +94,11 @@ configurations { distLib { visible = false } + jdeb +} + +dependencies { + jdeb group: 'org.vafer', name: 'jdeb', version: '0.8' } //task run(dependsOn: [configurations.distLib], description: 'Runs') << { @@ -182,7 +187,28 @@ task tar(type: Tar, dependsOn: ['explodedDist']) { } } -task release(dependsOn: [zip, tar]) << { + +task deb(dependsOn: ['explodedDist']) << { + ant.taskdef(name: "deb", classname: "org.vafer.jdeb.ant.DebAntTask", classpath: configurations.jdeb.asPath ) + ant.copy(todir: "${distsDir}/debian") { + fileset(dir: "pkg/debian/control") + filterset(begintoken: "[[", endtoken: "]]"){ + filter(token: "version", value: "${version}") + } + } + ant.deb(destfile: "${distsDir}/${archivesBaseName}-${version}-1_all.deb", control: "${distsDir}/debian", verbose: "true"){ + tarfileset(dir: explodedDistDir, prefix: "/usr/share/elasticsearch", includes: "*.txt, *.textile", username: "root", group: "root") + tarfileset(dir: explodedDistBinDir, prefix: "/usr/share/elasticsearch/bin", excludes: "*.bat", filemode: "755", username: "root", group: "root") + tarfileset(dir: explodedDistLibDir, prefix: "/usr/share/elasticsearch/lib", includes: "*.jar, sigar/*", username: "root", group: "root") + tarfileset(dir: explodedDistConfigDir, prefix: "/etc/elasticsearch", username: "root", group: "root") + tarfileset(dir: "pkg/debian/init.d", includes: "elasticsearch", prefix: "/etc/init.d", filemode: "755", username: "root", group: "root") + tarfileset(dir: "pkg/debian/default", includes: "elasticsearch", prefix: "/etc/default", username: "root", group: "root") + } + ant.delete(dir: "${distsDir}/debian") +} + + +task release(dependsOn: [zip, tar, deb]) << { ant.delete(dir: explodedDistDir) } diff --git a/pkg/debian/control/conffiles b/pkg/debian/control/conffiles new file mode 100644 index 0000000000000..98e0f5a64326c --- /dev/null +++ b/pkg/debian/control/conffiles @@ -0,0 +1,4 @@ +/etc/init.d/elasticsearch +/etc/default/elasticsearch +/etc/elasticsearch/logging.yml +/etc/elasticsearch/elasticsearch.yml diff --git a/pkg/debian/control/control b/pkg/debian/control/control new file mode 100644 index 0000000000000..d7026e6ee35ce --- /dev/null +++ b/pkg/debian/control/control @@ -0,0 +1,36 @@ +Package: elasticsearch +Version: [[version]] +Architecture: all +Maintainer: Nicolas Huray +Depends: openjdk-6-jre-headless | sun-java6-jre +Section: web +Priority: optional +Homepage: http://www.elasticsearch.org/ +Description: Open Source, Distributed, RESTful Search Engine + ElasticSearch is a distributed RESTful search engine built for the cloud. + . + Features include: + . + * Distributed and Highly Available Search Engine. + - Each index is fully sharded with a configurable number of shards. + - Each shard can have one or more replicas. + - Read / Search operations performed on either one of the replica shard. + * Multi Tenant with Multi Types. + - Support for more than one index. + - Support for more than one type per index. + - Index level configuration (number of shards, index storage, ...). + * Various set of APIs + - HTTP RESTful API + - Native Java API. + - All APIs perform automatic node operation rerouting. + * Document oriented + - No need for upfront schema definition. + - Schema can be defined per type for customization of the indexing process. + * Reliable, Asynchronous Write Behind for long term persistency. + * (Near) Real Time Search. + * Built on top of Lucene + - Each shard is a fully functional Lucene index + - All the power of Lucene easily exposed through simple configuration / plugins. + * Per operation consistency + - Single document level operations are atomic, consistent, isolated and durable. + * Open Source under Apache 2 License. diff --git a/pkg/debian/control/postinst b/pkg/debian/control/postinst new file mode 100755 index 0000000000000..85d631b6e23ac --- /dev/null +++ b/pkg/debian/control/postinst @@ -0,0 +1,39 @@ +#!/bin/sh +set -e + +case "$1" in + configure) + [ -f /etc/default/elasticsearch ] && . /etc/default/elasticsearch + [ -z "$ES_USER" ] && ES_USER="elasticsearch" + [ -z "$ES_GROUP" ] && ES_GROUP="elasticsearch" + if ! getent group "$ES_GROUP" > /dev/null 2>&1 ; then + addgroup --system "$ES_GROUP" --quiet + fi + if ! id $ES_USER > /dev/null 2>&1 ; then + adduser --system --home /usr/share/elasticsearch --no-create-home \ + --ingroup "$ES_GROUP" --disabled-password --shell /bin/false \ + "$ES_USER" + fi + + # Set user permissions on /var/log/elasticsearch and /var/lib/elasticsearch + mkdir -p /var/log/elasticsearch /var/lib/elasticsearch + chown -R $ES_USER:$ES_GROUP /var/log/elasticsearch /var/lib/elasticsearch + chmod 755 /var/log/elasticsearch /var/lib/elasticsearch + + # configuration files should not be modifiable by elasticsearch user, as this can be a security issue + chown -Rh root:root /etc/elasticsearch/* + chmod 755 /etc/elasticsearch + chmod 644 /etc/elasticsearch/* + ;; +esac + + +if [ -x "/etc/init.d/elasticsearch" ]; then + update-rc.d elasticsearch defaults 95 10 >/dev/null + if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then + invoke-rc.d elasticsearch start || true + else + /etc/init.d/elasticsearch start || true + fi +fi + diff --git a/pkg/debian/control/postrm b/pkg/debian/control/postrm new file mode 100755 index 0000000000000..776528a3af435 --- /dev/null +++ b/pkg/debian/control/postrm @@ -0,0 +1,30 @@ +#!/bin/sh +set -e + +case "$1" in + remove) + # Remove logs and data + rm -rf /var/log/elasticsearch /var/lib/elasticsearch + ;; + + purge) + # Remove service + update-rc.d elasticsearch remove >/dev/null || true + + # Remove logs and data + rm -rf /var/log/elasticsearch /var/lib/elasticsearch + + # Remove user/group + deluser elasticsearch || true + delgroup elasticsearch || true + ;; + + upgrade|failed-upgrade|abort-install|abort-upgrade|disappear) + # Nothing to do here + ;; + + *) + echo "$0 called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac diff --git a/pkg/debian/control/prerm b/pkg/debian/control/prerm new file mode 100755 index 0000000000000..f3603a24df70e --- /dev/null +++ b/pkg/debian/control/prerm @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +if [ -x "/etc/init.d/elasticsearch" ]; then + if [ -x "`which invoke-rc.d 2>/dev/null`" ]; then + invoke-rc.d elasticsearch stop || true + else + /etc/init.d/elasticsearch stop || true + fi +fi \ No newline at end of file diff --git a/pkg/debian/default/elasticsearch b/pkg/debian/default/elasticsearch new file mode 100644 index 0000000000000..f3cecf8af221b --- /dev/null +++ b/pkg/debian/default/elasticsearch @@ -0,0 +1,24 @@ +# Run ElasticSearch as this user ID and group ID +ES_USER=elasticsearch +ES_GROUP=elasticsearch + +# Minimum Heap memory to run ElasticSearch +ES_MIN_MEM=256m + +# Maximum Heap memory to run ElasticSearch +ES_MAX_MEM=1g + +# ElasticSearch log directory +LOG_DIR=/var/log/elasticsearch + +# ElasticSearch data directory +DATA_DIR=/var/lib/elasticsearch + +# ElasticSearch work directory +WORK_DIR=/tmp/elasticsearch + +# ElasticSearch configuration directory +CONF_DIR=/etc/elasticsearch + +# ElasticSearch configuration file (elasticsearch.yml) +CONF_FILE=/etc/elasticsearch/elasticsearch.yml diff --git a/pkg/debian/init.d/elasticsearch b/pkg/debian/init.d/elasticsearch new file mode 100644 index 0000000000000..61692194c31fe --- /dev/null +++ b/pkg/debian/init.d/elasticsearch @@ -0,0 +1,192 @@ +#!/bin/sh +# +# /etc/init.d/elasticsearch -- startup script for Elasticsearch +# +# Written by Miquel van Smoorenburg . +# Modified for Debian GNU/Linux by Ian Murdock . +# Modified for Tomcat by Stefan Gybas . +# Modified for Tomcat6 by Thierry Carrez . +# Additional improvements by Jason Brittain . +# Modified by Nicolas Huray for ElasticSearch . +# +### BEGIN INIT INFO +# Provides: elasticsearch +# Required-Start: $all +# Required-Stop: $all +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts elasticsearch +# Description: Starts elasticsearch using start-stop-daemon +### END INIT INFO + +set -e + +PATH=/bin:/usr/bin:/sbin:/usr/sbin +NAME=elasticsearch +DESC="ElasticSearch Server" +DEFAULT=/etc/default/$NAME + +if [ `id -u` -ne 0 ]; then + echo "You need root privileges to run this script" + exit 1 +fi + + +. /lib/lsb/init-functions + +if [ -r /etc/default/rcS ]; then + . /etc/default/rcS +fi + + +# The following variables can be overwritten in $DEFAULT + +# Run ElasticSearch as this user ID and group ID +ES_USER=elasticsearch +ES_GROUP=elasticsearch + +# The first existing directory is used for JAVA_HOME (if JAVA_HOME is not defined in $DEFAULT) +JDK_DIRS="/usr/lib/jvm/java-6-openjdk /usr/lib/jvm/java-6-sun" + +# Look for the right JVM to use +for jdir in $JDK_DIRS; do + if [ -r "$jdir/bin/java" -a -z "${JAVA_HOME}" ]; then + JAVA_HOME="$jdir" + fi +done +export JAVA_HOME + +# Directory where the ElasticSearch binary distribution resides +ES_HOME=/usr/share/$NAME + +# Minimum Heap memory to run ElasticSearch +ES_MIN_MEM=256m + +# Maximum Heap memory to run ElasticSearch +ES_MAX_MEM=1g + +# ElasticSearch log directory +LOG_DIR=/var/log/$NAME + +# ElasticSearch data directory +DATA_DIR=/var/lib/$NAME + +# ElasticSearch work directory +WORK_DIR=/tmp/$NAME + +# ElasticSearch configuration directory +CONF_DIR=/etc/$NAME + +# ElasticSearch configuration file (elasticsearch.yml) +CONF_FILE=$CONF_DIR/elasticsearch.yml + +# End of variables that can be overwritten in $DEFAULT + +# overwrite settings from default file +if [ -f "$DEFAULT" ]; then + . "$DEFAULT" +fi + +# Define other required variables +PID_FILE=/var/run/$NAME.pid +DAEMON=$ES_HOME/bin/elasticsearch +DAEMON_OPTS="-p $PID_FILE -Des.config=$CONF_FILE -Des.path.home=$ES_HOME -Des.path.logs=$LOG_DIR -Des.path.data=$DATA_DIR -Des.path.work=$WORK_DIR -Des.path.conf=$CONF_DIR" + +export ES_MIN_MEM ES_MAX_MEM + +# Check DAEMON exists +test -x $DAEMON || exit 0 + +case "$1" in + start) + if [ -z "$JAVA_HOME" ]; then + log_failure_msg "no JDK found - please set JAVA_HOME" + exit 1 + fi + + log_daemon_msg "Starting $DESC" + + if start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null; then + + # Prepare environment + mkdir -p "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" && chown "$ES_USER":"$ES_GROUP" "$LOG_DIR" "$DATA_DIR" "$WORK_DIR" + touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE" + ulimit -n 65535 + + # Start Daemon + start-stop-daemon --start -b --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec /bin/bash -- -c "$DAEMON $DAEMON_OPTS" + + sleep 1 + if start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null; then + if [ -f "$PID_FILE" ]; then + rm -f "$PID_FILE" + fi + log_end_msg 1 + else + log_end_msg 0 + fi + + else + log_progress_msg "(already running)" + log_end_msg 0 + fi + ;; + stop) + log_daemon_msg "Stopping $DESC" + + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" \ + --user "$ES_USER" \ + --retry=TERM/20/KILL/5 >/dev/null + if [ $? -eq 1 ]; then + log_progress_msg "$DESC is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop $DESC (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_progress_msg "(not running)" + fi + log_end_msg 0 + set -e + ;; + status) + set +e + start-stop-daemon --test --start --pidfile "$PID_FILE" \ + --user "$ES_USER" --exec "$JAVA_HOME/bin/java" \ + >/dev/null 2>&1 + if [ "$?" = "0" ]; then + + if [ -f "$PID_FILE" ]; then + log_success_msg "$DESC is not running, but pid file exists." + exit 1 + else + log_success_msg "$DESC is not running." + exit 3 + fi + else + log_success_msg "$DESC is running with pid `cat $PID_FILE`" + fi + set -e + ;; + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + *) + log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" + exit 1 + ;; +esac + +exit 0 From 56a4c98e8b4c26771ce7424d4fc758643293dab0 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 3 Oct 2011 12:13:02 +0200 Subject: [PATCH 88/96] Allow to specify a specific field in the clear cache API, closes #1374. --- .../cache/clear/ClearIndicesCacheRequest.java | 25 +++++++++++++++++++ .../clear/ShardClearIndicesCacheRequest.java | 21 ++++++++++++++++ .../TransportClearIndicesCacheAction.java | 17 +++++++++++-- .../ClearIndicesCacheRequestBuilder.java | 5 ++++ .../cache/field/data/FieldDataCache.java | 2 ++ .../field/data/none/NoneFieldDataCache.java | 4 +++ .../AbstractConcurrentMapFieldDataCache.java | 7 ++++++ .../clear/RestClearIndicesCacheAction.java | 8 +++++- 8 files changed, 86 insertions(+), 3 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java index 51343d9de0296..7704af11de9b1 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java @@ -35,6 +35,7 @@ public class ClearIndicesCacheRequest extends BroadcastOperationRequest { private boolean fieldDataCache = false; private boolean idCache = false; private boolean bloomCache = false; + private String[] fields = null; ClearIndicesCacheRequest() { } @@ -79,6 +80,15 @@ public ClearIndicesCacheRequest fieldDataCache(boolean fieldDataCache) { return this; } + public ClearIndicesCacheRequest fields(String... fields) { + this.fields = fields; + return this; + } + + public String[] fields() { + return this.fields; + } + public boolean idCache() { return this.idCache; } @@ -103,6 +113,13 @@ public void readFrom(StreamInput in) throws IOException { fieldDataCache = in.readBoolean(); idCache = in.readBoolean(); bloomCache = in.readBoolean(); + int size = in.readVInt(); + if (size > 0) { + fields = new String[size]; + for (int i = 0; i < size; i++) { + fields[i] = in.readUTF(); + } + } } public void writeTo(StreamOutput out) throws IOException { @@ -111,5 +128,13 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(fieldDataCache); out.writeBoolean(idCache); out.writeBoolean(bloomCache); + if (fields == null) { + out.writeVInt(0); + } else { + out.writeVInt(fields.length); + for (String field : fields) { + out.writeUTF(field); + } + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheRequest.java index 78509995efd87..3ec0976ac6155 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/ShardClearIndicesCacheRequest.java @@ -34,6 +34,7 @@ class ShardClearIndicesCacheRequest extends BroadcastShardOperationRequest { private boolean fieldDataCache = false; private boolean idCache = false; private boolean bloomCache = false; + private String[] fields = null; ShardClearIndicesCacheRequest() { } @@ -44,6 +45,7 @@ public ShardClearIndicesCacheRequest(String index, int shardId, ClearIndicesCach fieldDataCache = request.fieldDataCache(); idCache = request.idCache(); bloomCache = request.bloomCache(); + fields = request.fields(); } public boolean filterCache() { @@ -62,6 +64,10 @@ public boolean bloomCache() { return this.bloomCache; } + public String[] fields() { + return this.fields; + } + public ShardClearIndicesCacheRequest waitForOperations(boolean waitForOperations) { this.filterCache = waitForOperations; return this; @@ -73,6 +79,13 @@ public ShardClearIndicesCacheRequest waitForOperations(boolean waitForOperations fieldDataCache = in.readBoolean(); idCache = in.readBoolean(); bloomCache = in.readBoolean(); + int size = in.readVInt(); + if (size > 0) { + fields = new String[size]; + for (int i = 0; i < size; i++) { + fields[i] = in.readUTF(); + } + } } @Override public void writeTo(StreamOutput out) throws IOException { @@ -81,5 +94,13 @@ public ShardClearIndicesCacheRequest waitForOperations(boolean waitForOperations out.writeBoolean(fieldDataCache); out.writeBoolean(idCache); out.writeBoolean(bloomCache); + if (fields == null) { + out.writeVInt(0); + } else { + out.writeVInt(fields.length); + for (String field : fields) { + out.writeUTF(field); + } + } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java index b06382101e42d..9ee04075109d0 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java @@ -122,7 +122,13 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio } if (request.fieldDataCache()) { clearedAtLeastOne = true; - service.cache().fieldData().clear(); + if (request.fields() == null || request.fields().length == 0) { + service.cache().fieldData().clear(); + } else { + for (String field : request.fields()) { + service.cache().fieldData().clear(field); + } + } } if (request.idCache()) { clearedAtLeastOne = true; @@ -133,7 +139,14 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastOperatio service.cache().bloomCache().clear(); } if (!clearedAtLeastOne) { - service.cache().clear(); + if (request.fields() != null && request.fields().length > 0) { + // only clear caches relating to the specified fields + for (String field : request.fields()) { + service.cache().fieldData().clear(field); + } + } else { + service.cache().clear(); + } } service.cache().invalidateCache(); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java index 6ba23b537231b..c232c4cdaee2c 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java @@ -50,6 +50,11 @@ public ClearIndicesCacheRequestBuilder setFieldDataCache(boolean fieldDataCache) return this; } + public ClearIndicesCacheRequestBuilder setFields(String... fields) { + request.fields(fields); + return this; + } + public ClearIndicesCacheRequestBuilder setIdCache(boolean idCache) { request.idCache(idCache); return this; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java index 09e4c76a488c5..835b24dc3fd60 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/FieldDataCache.java @@ -36,6 +36,8 @@ public interface FieldDataCache extends IndexComponent, CloseableComponent { String type(); + void clear(String fieldName); + void clear(); void clear(IndexReader reader); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java index 11aab174ac458..a0393edda2967 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/none/NoneFieldDataCache.java @@ -50,6 +50,10 @@ public class NoneFieldDataCache extends AbstractIndexComponent implements FieldD return "none"; } + @Override public void clear(String fieldName) { + + } + @Override public void clear() { } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java index fa455f7c47c12..bd17cf2847f17 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/cache/field/data/support/AbstractConcurrentMapFieldDataCache.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.settings.IndexSettings; import java.io.IOException; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -54,6 +55,12 @@ protected AbstractConcurrentMapFieldDataCache(Index index, @IndexSettings Settin clear(); } + @Override public void clear(String fieldName) { + for (Map.Entry> entry : cache.entrySet()) { + entry.getValue().remove(fieldName); + } + } + @Override public void clear() { cache.clear(); } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java index 25843e97e7b42..c9836b52e186b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/admin/indices/cache/clear/RestClearIndicesCacheAction.java @@ -27,7 +27,12 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import org.elasticsearch.rest.action.support.RestActions; import org.elasticsearch.rest.action.support.RestXContentBuilder; @@ -58,6 +63,7 @@ public class RestClearIndicesCacheAction extends BaseRestHandler { clearIndicesCacheRequest.fieldDataCache(request.paramAsBoolean("field_data", clearIndicesCacheRequest.fieldDataCache())); clearIndicesCacheRequest.idCache(request.paramAsBoolean("id", clearIndicesCacheRequest.idCache())); clearIndicesCacheRequest.bloomCache(request.paramAsBoolean("bloom", clearIndicesCacheRequest.bloomCache())); + clearIndicesCacheRequest.fields(request.paramAsStringArray("fields", clearIndicesCacheRequest.fields())); // we just send back a response, no need to fork a listener clearIndicesCacheRequest.listenerThreaded(false); From a51baa7d6c86df8aec50eb071678fb4f4aa8016d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 2 Oct 2011 22:26:30 -0400 Subject: [PATCH 89/96] Allow limiting the number of concurrent ES nodes that can run on the same machine --- .../src/main/java/org/elasticsearch/env/NodeEnvironment.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 3a4c273f67385..60a8be3a88001 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -60,7 +60,8 @@ public class NodeEnvironment extends AbstractComponent { Lock[] locks = new Lock[environment.dataWithClusterFiles().length]; int localNodeId = -1; IOException lastException = null; - for (int possibleLockId = 0; possibleLockId < 50; possibleLockId++) { + int maxLocalStorageNodes = settings.getAsInt("node.max_local_storage_nodes", 50); + for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) { for (int dirIndex = 0; dirIndex < environment.dataWithClusterFiles().length; dirIndex++) { File dir = new File(new File(environment.dataWithClusterFiles()[dirIndex], "nodes"), Integer.toString(possibleLockId)); if (!dir.exists()) { From 3d49b4ed3a1bb083387fbf4b5c3d9dea81c6e1ae Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Mon, 3 Oct 2011 13:48:10 +0200 Subject: [PATCH 90/96] add a simple integration level test for awareness --- .../allocation/AwarenessAllocationTests.java | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/AwarenessAllocationTests.java diff --git a/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/AwarenessAllocationTests.java b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/AwarenessAllocationTests.java new file mode 100644 index 0000000000000..ca911c0010f72 --- /dev/null +++ b/modules/test/integration/src/test/java/org/elasticsearch/test/integration/cluster/allocation/AwarenessAllocationTests.java @@ -0,0 +1,88 @@ +/* + * Licensed to Elastic Search and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Elastic Search licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.cluster.allocation; + +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.trove.map.hash.TObjectIntHashMap; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.Test; + +import static org.hamcrest.MatcherAssert.*; +import static org.hamcrest.Matchers.*; + +/** + */ +@Test +public class AwarenessAllocationTests extends AbstractNodesTests { + + private final ESLogger logger = Loggers.getLogger(AwarenessAllocationTests.class); + + @AfterMethod public void cleanAndCloseNodes() throws Exception { + closeAllNodes(); + } + + @Test public void testSimpleAwareness() throws Exception { + Settings commonSettings = ImmutableSettings.settingsBuilder() + .put("cluster.routing.schedule", "10ms") + .put("cluster.routing.allocation.awareness.attributes", "rack_id") + .build(); + + + logger.info("--> starting 2 nodes on the same rack"); + startNode("node1", ImmutableSettings.settingsBuilder().put(commonSettings).put("node.rack_id", "rack_1")); + startNode("node2", ImmutableSettings.settingsBuilder().put(commonSettings).put("node.rack_id", "rack_1")); + + client("node1").admin().indices().prepareCreate("test1").execute().actionGet(); + client("node1").admin().indices().prepareCreate("test2").execute().actionGet(); + + ClusterHealthResponse health = client("node1").admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet(); + assertThat(health.timedOut(), equalTo(false)); + + logger.info("--> starting 1 node on a different rack"); + startNode("node3", ImmutableSettings.settingsBuilder().put(commonSettings).put("node.rack_id", "rack_2")); + + Thread.sleep(500); + + health = client("node1").admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("3").setWaitForRelocatingShards(0).execute().actionGet(); + assertThat(health.timedOut(), equalTo(false)); + + ClusterState clusterState = client("node1").admin().cluster().prepareState().execute().actionGet().state(); + //System.out.println(clusterState.routingTable().prettyPrint()); + // verify that we have 10 shards on node3 + TObjectIntHashMap counts = new TObjectIntHashMap(); + for (IndexRoutingTable indexRoutingTable : clusterState.routingTable()) { + for (IndexShardRoutingTable indexShardRoutingTable : indexRoutingTable) { + for (ShardRouting shardRouting : indexShardRoutingTable) { + counts.adjustOrPutValue(clusterState.nodes().get(shardRouting.currentNodeId()).name(), 1, 1); + } + } + } + assertThat(counts.get("node3"), equalTo(10)); + } +} From 3d4c31de91a0cbdceb4c910a5bd8eaad3bd09501 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 12:42:27 +0200 Subject: [PATCH 91/96] associate a version with a discovery node --- .../main/java/org/elasticsearch/Version.java | 100 ++++++++++++------ .../elasticsearch/bootstrap/Bootstrap.java | 2 +- .../cluster/node/DiscoveryNode.java | 13 +++ .../node/internal/InternalNode.java | 16 +-- .../elasticsearch/plugins/PluginManager.java | 10 +- .../rest/action/main/RestMainAction.java | 11 +- .../memcached/netty/MemcachedDecoder.java | 2 +- 7 files changed, 103 insertions(+), 51 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/Version.java b/modules/elasticsearch/src/main/java/org/elasticsearch/Version.java index 7ab7fcf19b990..440411410bd36 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/Version.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/Version.java @@ -19,63 +19,95 @@ package org.elasticsearch; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.monitor.jvm.JvmInfo; -import java.io.InputStream; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Properties; -import java.util.TimeZone; +import java.io.IOException; /** - * @author kimchy (shay.banon) */ public class Version { - private static final String number; - private static final String date; - private static final boolean snapshotBuild; + // The logic for ID is: XXYYZZAA, where XX is major version, YY is minor version, ZZ is revision, and AA is Beta/RC indicator + // AA values below 50 are beta builds, and below 99 are RC builds, with 99 indicating a release + // the (internal) format of the id is there so we can easily do after/before checks on the id + public static final int V_0_18_0_ID = /*00*/180099; + public static final Version V_0_18_0 = new Version(V_0_18_0_ID, true); - static { - Properties props = new Properties(); - try { - InputStream stream = Version.class.getClassLoader().getResourceAsStream("org/elasticsearch/version.properties"); - props.load(stream); - stream.close(); - } catch (Exception e) { - // ignore + public static final Version CURRENT = V_0_18_0; + + public static Version readVersion(StreamInput in) throws IOException { + return fromId(in.readVInt()); + } + + private static Version fromId(int id) { + switch (id) { + case V_0_18_0_ID: + return V_0_18_0; + default: + return new Version(id, null); } + } - number = props.getProperty("number", "0.0.0"); - snapshotBuild = number.contains("-SNAPSHOT"); - SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); - sdf.setTimeZone(TimeZone.getTimeZone("UTC")); - date = props.getProperty("date", sdf.format(new Date())); + public static void writeVersion(Version version, StreamOutput out) throws IOException { + out.writeVInt(version.id); } - public static String number() { - return number; + public final int id; + public final byte major; + public final byte minor; + public final byte revision; + public final byte build; + public final Boolean snapshot; + + Version(int id, @Nullable Boolean snapshot) { + this.id = id; + this.major = (byte) ((id / 1000000) % 100); + this.minor = (byte) ((id / 10000) % 100); + this.revision = (byte) ((id / 100) % 100); + this.build = (byte) (id % 100); + this.snapshot = snapshot; } - public static String date() { - return date; + public boolean snapshot() { + return snapshot != null && snapshot; } - public static boolean snapshotBuild() { - return snapshotBuild; + public boolean after(Version version) { + return version.id > id; } - public static String full() { - StringBuilder sb = new StringBuilder("elasticsearch/"); - sb.append(number); - if (snapshotBuild) { - sb.append("/").append(date); + public boolean onOrAfter(Version version) { + return version.id >= id; + } + + /** + * Just the version number (without -SNAPSHOT if snapshot). + */ + public String number() { + StringBuilder sb = new StringBuilder(); + sb.append(major).append('.').append(minor).append('.').append(revision); + if (build < 50) { + sb.append(".Beta").append(build); + } else if (build < 99) { + sb.append(".RC").append(build - 50); } return sb.toString(); } public static void main(String[] args) { - System.out.println("ElasticSearch Version: " + number + " (" + date() + "), JVM: " + JvmInfo.jvmInfo().vmVersion()); + System.out.println("ElasticSearch Version: " + Version.CURRENT + "JVM: " + JvmInfo.jvmInfo().vmVersion()); + } + + @Override public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(number()); + if (snapshot()) { + sb.append("-SNAPSHOT"); + } + return sb.toString(); } } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/modules/elasticsearch/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index 7ad9f78e20f29..5a47a4933c8d3 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -235,7 +235,7 @@ public static void main(String[] args) { } private static String buildErrorMessage(String stage, Throwable e) { - StringBuilder errorMessage = new StringBuilder("{").append(Version.full()).append("}: "); + StringBuilder errorMessage = new StringBuilder("{").append(Version.CURRENT).append("}: "); try { if (ANSI.isEnabled()) { errorMessage.append(attrib(ANSI.Code.FG_RED)).append(stage).append(" Failed ...").append(attrib(ANSI.Code.OFF)).append("\n"); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java index f741a3f11b58e..7f457bde07998 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/cluster/node/DiscoveryNode.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.node; +import org.elasticsearch.Version; import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.io.stream.StreamInput; @@ -55,6 +56,8 @@ public static boolean nodeRequiresLocalStorage(Settings settings) { private ImmutableMap attributes; + private Version version = Version.CURRENT; + private DiscoveryNode() { } @@ -191,6 +194,14 @@ public boolean isMasterNode() { return masterNode(); } + public Version version() { + return this.version; + } + + public Version getVersion() { + return this.version; + } + public static DiscoveryNode readNode(StreamInput in) throws IOException { DiscoveryNode node = new DiscoveryNode(); node.readFrom(in); @@ -207,6 +218,7 @@ public static DiscoveryNode readNode(StreamInput in) throws IOException { builder.put(in.readUTF().intern(), in.readUTF().intern()); } attributes = builder.build(); + version = Version.readVersion(in); } @Override public void writeTo(StreamOutput out) throws IOException { @@ -218,6 +230,7 @@ public static DiscoveryNode readNode(StreamInput in) throws IOException { out.writeUTF(entry.getKey()); out.writeUTF(entry.getValue()); } + Version.writeVersion(version, out); } @Override public boolean equals(Object obj) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java index a5dfeade20702..c787558f0dfcc 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/node/internal/InternalNode.java @@ -112,7 +112,7 @@ public InternalNode(Settings pSettings, boolean loadConfigSettings) throws Elast Tuple tuple = InternalSettingsPerparer.prepareSettings(pSettings, loadConfigSettings); ESLogger logger = Loggers.getLogger(Node.class, tuple.v1().get("name")); - logger.info("{{}}[{}]: initializing ...", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: initializing ...", Version.CURRENT, JvmInfo.jvmInfo().pid()); this.pluginsService = new PluginsService(tuple.v1(), tuple.v2()); this.settings = pluginsService.updatedSettings(); @@ -149,7 +149,7 @@ public InternalNode(Settings pSettings, boolean loadConfigSettings) throws Elast client = injector.getInstance(Client.class); - logger.info("{{}}[{}]: initialized", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: initialized", Version.CURRENT, JvmInfo.jvmInfo().pid()); } @Override public Settings settings() { @@ -166,7 +166,7 @@ public Node start() { } ESLogger logger = Loggers.getLogger(Node.class, settings.get("name")); - logger.info("{{}}[{}]: starting ...", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: starting ...", Version.CURRENT, JvmInfo.jvmInfo().pid()); for (Class plugin : pluginsService.services()) { injector.getInstance(plugin).start(); @@ -193,7 +193,7 @@ public Node start() { } injector.getInstance(JmxService.class).connectAndRegister(discoService.nodeDescription(), injector.getInstance(NetworkService.class)); - logger.info("{{}}[{}]: started", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: started", Version.CURRENT, JvmInfo.jvmInfo().pid()); return this; } @@ -203,7 +203,7 @@ public Node start() { return this; } ESLogger logger = Loggers.getLogger(Node.class, settings.get("name")); - logger.info("{{}}[{}]: stopping ...", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: stopping ...", Version.CURRENT, JvmInfo.jvmInfo().pid()); if (settings.getAsBoolean("http.enabled", true)) { injector.getInstance(HttpServer.class).stop(); @@ -237,7 +237,7 @@ public Node start() { injector.getInstance(plugin).stop(); } - logger.info("{{}}[{}]: stopped", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: stopped", Version.CURRENT, JvmInfo.jvmInfo().pid()); return this; } @@ -251,7 +251,7 @@ public void close() { } ESLogger logger = Loggers.getLogger(Node.class, settings.get("name")); - logger.info("{{}}[{}]: closing ...", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: closing ...", Version.CURRENT, JvmInfo.jvmInfo().pid()); StopWatch stopWatch = new StopWatch("node_close"); stopWatch.start("http"); @@ -325,7 +325,7 @@ public void close() { injector.getInstance(NodeEnvironment.class).close(); Injectors.close(injector); - logger.info("{{}}[{}]: closed", Version.full(), JvmInfo.jvmInfo().pid()); + logger.info("{{}}[{}]: closed", Version.CURRENT, JvmInfo.jvmInfo().pid()); } @Override public boolean isClosed() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java index f89f324d52e96..472b5c3f79d58 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -65,11 +65,11 @@ public void checkServerTrusted( public void downloadAndExtract(String name) throws IOException { HttpDownloadHelper downloadHelper = new HttpDownloadHelper(); - File pluginFile = new File(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip"); + File pluginFile = new File(url + "/" + name + "/elasticsearch-" + name + "-" + Version.CURRENT.number() + ".zip"); boolean downloaded = false; String filterZipName = null; if (!pluginFile.exists()) { - pluginFile = new File(url + "/elasticsearch-" + name + "-" + Version.number() + ".zip"); + pluginFile = new File(url + "/elasticsearch-" + name + "-" + Version.CURRENT.number() + ".zip"); if (!pluginFile.exists()) { pluginFile = new File(environment.pluginsFile(), name + ".zip"); if (url != null) { @@ -107,14 +107,14 @@ public void downloadAndExtract(String name) throws IOException { pluginFile = new File(environment.pluginsFile(), name + ".zip"); if (version == null) { // try with ES version from downloads - URL pluginUrl = new URL("https://github.com/downloads/" + userName + "/" + repoName + "/" + repoName + "-" + Version.number() + ".zip"); + URL pluginUrl = new URL("https://github.com/downloads/" + userName + "/" + repoName + "/" + repoName + "-" + Version.CURRENT.number() + ".zip"); System.out.println("Trying " + pluginUrl.toExternalForm() + "..."); try { downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out)); downloaded = true; } catch (IOException e) { // try a tag with ES version - pluginUrl = new URL("https://github.com/" + userName + "/" + repoName + "/zipball/v" + Version.number()); + pluginUrl = new URL("https://github.com/" + userName + "/" + repoName + "/zipball/v" + Version.CURRENT.number()); System.out.println("Trying " + pluginUrl.toExternalForm() + "..."); try { downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out)); @@ -151,7 +151,7 @@ public void downloadAndExtract(String name) throws IOException { } } } else { - URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.number() + ".zip"); + URL pluginUrl = new URL(url + "/" + name + "/elasticsearch-" + name + "-" + Version.CURRENT.number() + ".zip"); System.out.println("Trying " + pluginUrl.toExternalForm() + "..."); try { downloadHelper.download(pluginUrl, pluginFile, new HttpDownloadHelper.VerboseProgress(System.out)); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java index 5c454fb94d0f1..edc0ae32c4a6d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/main/RestMainAction.java @@ -29,7 +29,14 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.StringRestResponse; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import org.elasticsearch.rest.action.support.RestXContentBuilder; import java.io.IOException; @@ -79,7 +86,7 @@ public class RestMainAction extends BaseRestHandler { if (settings.get("name") != null) { builder.field("name", settings.get("name")); } - builder.startObject("version").field("number", Version.number()).field("date", Version.date()).field("snapshot_build", Version.snapshotBuild()).endObject(); + builder.startObject("version").field("number", Version.CURRENT.number()).field("snapshot_build", Version.CURRENT.snapshot).endObject(); builder.field("tagline", "You Know, for Search"); builder.field("cover", "DON'T PANIC"); if (rootNode != null) { diff --git a/plugins/transport/memcached/src/main/java/org/elasticsearch/memcached/netty/MemcachedDecoder.java b/plugins/transport/memcached/src/main/java/org/elasticsearch/memcached/netty/MemcachedDecoder.java index e8ea51473d1f1..8db4da4288c9e 100644 --- a/plugins/transport/memcached/src/main/java/org/elasticsearch/memcached/netty/MemcachedDecoder.java +++ b/plugins/transport/memcached/src/main/java/org/elasticsearch/memcached/netty/MemcachedDecoder.java @@ -185,7 +185,7 @@ public MemcachedDecoder(ESLogger logger) { this.request = new MemcachedRestRequest(RestRequest.Method.POST, args[1], null, Integer.parseInt(args[4]), false); buffer.markReaderIndex(); } else if ("version".equals(cmd)) { // sent as a noop - byte[] bytes = Version.full().getBytes(); + byte[] bytes = Version.CURRENT.toString().getBytes(); ChannelBuffer writeBuffer = ChannelBuffers.dynamicBuffer(bytes.length); writeBuffer.writeBytes(bytes); channel.write(writeBuffer); From 63844ddd4315d5d979660501910651094eecfc5a Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 13:03:30 +0200 Subject: [PATCH 92/96] REST Bulk API: Allow to execute _bulk against /{index}/_bulk and /{index}/{type}/_bulk endpoints, closes #1375. --- .../elasticsearch/action/bulk/BulkRequest.java | 18 +++++++++++++----- .../client/action/bulk/BulkRequestBuilder.java | 11 ++++++++++- .../rest/action/bulk/RestBulkAction.java | 15 +++++++++++++-- .../action/bulk/BulkActionTests.java | 12 ++++++++++-- .../action/bulk/simple-bulk2.json | 5 +++++ 5 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk2.json diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 9ceeeeec30779..89ce92be922dc 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.replication.ReplicationType; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.Lists; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -82,6 +83,13 @@ public BulkRequest add(DeleteRequest request) { * Adds a framed data in binary format */ public BulkRequest add(byte[] data, int from, int length, boolean contentUnsafe) throws Exception { + return add(data, from, length, contentUnsafe, null, null); + } + + /** + * Adds a framed data in binary format + */ + public BulkRequest add(byte[] data, int from, int length, boolean contentUnsafe, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception { XContent xContent = XContentFactory.xContent(data, from, length); byte marker = xContent.streamSeparator(); while (true) { @@ -105,12 +113,9 @@ public BulkRequest add(byte[] data, int from, int length, boolean contentUnsafe) token = parser.nextToken(); assert token == XContentParser.Token.FIELD_NAME; String action = parser.currentName(); - // Move to START_OBJECT - token = parser.nextToken(); - assert token == XContentParser.Token.START_OBJECT; - String index = null; - String type = null; + String index = defaultIndex; + String type = defaultType; String id = null; String routing = null; String parent = null; @@ -121,6 +126,9 @@ public BulkRequest add(byte[] data, int from, int length, boolean contentUnsafe) VersionType versionType = VersionType.INTERNAL; String percolate = null; + // at this stage, next token can either be END_OBJECT (and use default index and type, with auto generated id) + // or START_OBJECT which will have another set of parameters + String currentFieldName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java index 83fa249e6b556..b4b051f9dbda9 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/client/action/bulk/BulkRequestBuilder.java @@ -30,6 +30,7 @@ import org.elasticsearch.client.action.delete.DeleteRequestBuilder; import org.elasticsearch.client.action.index.IndexRequestBuilder; import org.elasticsearch.client.action.support.BaseRequestBuilder; +import org.elasticsearch.common.Nullable; /** * A bulk request holds an ordered {@link IndexRequest}s and {@link DeleteRequest}s and allows to executes @@ -81,7 +82,15 @@ public BulkRequestBuilder add(DeleteRequestBuilder request) { * Adds a framed data in binary format */ public BulkRequestBuilder add(byte[] data, int from, int length, boolean contentUnsafe) throws Exception { - request.add(data, from, length, contentUnsafe); + request.add(data, from, length, contentUnsafe, null, null); + return this; + } + + /** + * Adds a framed data in binary format + */ + public BulkRequestBuilder add(byte[] data, int from, int length, boolean contentUnsafe, @Nullable String defaultIndex, @Nullable String defaultType) throws Exception { + request.add(data, from, length, contentUnsafe, defaultIndex, defaultType); return this; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java index 166a7981999e1..916607b25af34 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/rest/action/bulk/RestBulkAction.java @@ -32,7 +32,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.elasticsearch.rest.*; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestChannel; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.XContentRestResponse; +import org.elasticsearch.rest.XContentThrowableRestResponse; import java.io.IOException; @@ -58,10 +63,16 @@ public class RestBulkAction extends BaseRestHandler { controller.registerHandler(POST, "/_bulk", this); controller.registerHandler(PUT, "/_bulk", this); + controller.registerHandler(POST, "/{index}/_bulk", this); + controller.registerHandler(PUT, "/{index}/_bulk", this); + controller.registerHandler(POST, "/{index}/{type}/_bulk", this); + controller.registerHandler(PUT, "/{index}/{type}/_bulk", this); } @Override public void handleRequest(final RestRequest request, final RestChannel channel) { BulkRequest bulkRequest = Requests.bulkRequest(); + String defaultIndex = request.param("index"); + String defaultType = request.param("type"); String replicationType = request.param("replication"); if (replicationType != null) { @@ -73,7 +84,7 @@ public class RestBulkAction extends BaseRestHandler { } bulkRequest.refresh(request.paramAsBoolean("refresh", bulkRequest.refresh())); try { - bulkRequest.add(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), request.contentUnsafe()); + bulkRequest.add(request.contentByteArray(), request.contentByteArrayOffset(), request.contentLength(), request.contentUnsafe(), defaultIndex, defaultType); } catch (Exception e) { try { XContentBuilder builder = restContentBuilder(request); diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java index a9cb2644e623c..b98253cbdebb1 100644 --- a/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/BulkActionTests.java @@ -26,10 +26,18 @@ import static org.hamcrest.Matchers.*; public class BulkActionTests { - @Test public void testSimpleBulk() throws Exception { + + @Test public void testSimpleBulk1() throws Exception { String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk.json"); BulkRequest bulkRequest = new BulkRequest(); - bulkRequest.add(bulkAction.getBytes(), 0, bulkAction.length(), true); + bulkRequest.add(bulkAction.getBytes(), 0, bulkAction.length(), true, null, null); + assertThat(bulkRequest.numberOfActions(), equalTo(3)); + } + + @Test public void testSimpleBulk2() throws Exception { + String bulkAction = copyToStringFromClasspath("/org/elasticsearch/action/bulk/simple-bulk2.json"); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(bulkAction.getBytes(), 0, bulkAction.length(), true, null, null); assertThat(bulkRequest.numberOfActions(), equalTo(3)); } } diff --git a/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk2.json b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk2.json new file mode 100644 index 0000000000000..ccb621d448894 --- /dev/null +++ b/modules/elasticsearch/src/test/java/org/elasticsearch/action/bulk/simple-bulk2.json @@ -0,0 +1,5 @@ +{ "index" : { } } +{ "field1" : "value1" } +{ "delete" : { "_id" : "2" } } +{ "create" : { "_id" : "3" } } +{ "field1" : "value3" } From 111c472a0b7647b9c6a82e472ff69f9010a88df4 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 13:32:33 +0200 Subject: [PATCH 93/96] upgrade to jackson 1.9.0 --- modules/jarjar/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/jarjar/build.gradle b/modules/jarjar/build.gradle index c6e09be290de9..2f9aa8bd1a2af 100644 --- a/modules/jarjar/build.gradle +++ b/modules/jarjar/build.gradle @@ -15,8 +15,8 @@ dependencies { runtime 'joda-time:joda-time:1.6.2' runtime 'org.mvel:mvel2:2.1.Beta6' - runtime 'org.codehaus.jackson:jackson-core-asl:1.8.5' - runtime 'org.codehaus.jackson:jackson-smile:1.8.5' + runtime 'org.codehaus.jackson:jackson-core-asl:1.9.0' + runtime 'org.codehaus.jackson:jackson-smile:1.9.0' runtime 'org.yaml:snakeyaml:1.6' runtime('org.jboss.netty:netty:3.2.5.Final') { transitive = false } From 5c783c8ef1d35ac0201ef25227a9416bf30ead92 Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 15:21:56 +0200 Subject: [PATCH 94/96] Upgrade to Apache Tika 0.10, closes #1372. --- .idea/modules/plugin-mapper-attachments.iml | 2 +- plugins/mapper/attachments/build.gradle | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/modules/plugin-mapper-attachments.iml b/.idea/modules/plugin-mapper-attachments.iml index 812f6f4526c2b..4a67d2f6ef3d9 100644 --- a/.idea/modules/plugin-mapper-attachments.iml +++ b/.idea/modules/plugin-mapper-attachments.iml @@ -16,7 +16,7 @@ - + diff --git a/plugins/mapper/attachments/build.gradle b/plugins/mapper/attachments/build.gradle index eb05dd16d75ed..6b5652141325a 100644 --- a/plugins/mapper/attachments/build.gradle +++ b/plugins/mapper/attachments/build.gradle @@ -33,8 +33,8 @@ configurations { dependencies { compile project(':elasticsearch') - compile('org.apache.tika:tika-app:0.9') { transitive = false } - distLib('org.apache.tika:tika-app:0.9') { transitive = false } + compile('org.apache.tika:tika-app:0.10') { transitive = false } + distLib('org.apache.tika:tika-app:0.10') { transitive = false } } task explodedDist(dependsOn: [jar], description: 'Builds the plugin zip file') << { From 8f55e8835f7b3508f4783e34ea4243b9cb85596f Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 17:29:11 +0200 Subject: [PATCH 95/96] when getting the _meta doc from _river index (per river), make sure the callback listener is threaded --- .../src/main/java/org/elasticsearch/river/RiversService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java index 3cecddc9003ec..88d9247494625 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java @@ -218,7 +218,7 @@ private class ApplyRivers implements RiverClusterStateListener { if (rivers.containsKey(routing.riverName())) { continue; } - client.prepareGet(riverIndexName, routing.riverName().name(), "_meta").execute(new ActionListener() { + client.prepareGet(riverIndexName, routing.riverName().name(), "_meta").setListenerThreaded(true).execute(new ActionListener() { @Override public void onResponse(GetResponse getResponse) { if (!rivers.containsKey(routing.riverName())) { if (getResponse.exists()) { @@ -238,7 +238,7 @@ private class ApplyRivers implements RiverClusterStateListener { final ActionListener listener = this; threadPool.schedule(TimeValue.timeValueSeconds(5), ThreadPool.Names.SAME, new Runnable() { @Override public void run() { - client.prepareGet(riverIndexName, routing.riverName().name(), "_meta").execute(listener); + client.prepareGet(riverIndexName, routing.riverName().name(), "_meta").setListenerThreaded(true).execute(listener); } }); } else { From d69baa3e04407380972620c62286c2cd9bab06be Mon Sep 17 00:00:00 2001 From: Shay Banon Date: Wed, 5 Oct 2011 19:12:21 +0200 Subject: [PATCH 96/96] when writing _river status, use consistency of one --- .../main/java/org/elasticsearch/river/RiversService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java b/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java index 88d9247494625..2c1bcb270f64e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/river/RiversService.java @@ -23,6 +23,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; +import org.elasticsearch.action.WriteConsistencyLevel; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterService; @@ -150,7 +151,9 @@ public synchronized void createRiver(RiverName riverName, Map se builder.endObject(); - client.prepareIndex(riverIndexName, riverName.name(), "_status").setSource(builder).execute().actionGet(); + client.prepareIndex(riverIndexName, riverName.name(), "_status") + .setConsistencyLevel(WriteConsistencyLevel.ONE) + .setSource(builder).execute().actionGet(); } catch (Exception e) { logger.warn("failed to create river [{}][{}]", e, riverName.type(), riverName.name()); @@ -164,7 +167,9 @@ public synchronized void createRiver(RiverName riverName, Map se builder.field("transport_address", clusterService.localNode().address().toString()); builder.endObject(); - client.prepareIndex(riverIndexName, riverName.name(), "_status").setSource(builder).execute().actionGet(); + client.prepareIndex(riverIndexName, riverName.name(), "_status") + .setConsistencyLevel(WriteConsistencyLevel.ONE) + .setSource(builder).execute().actionGet(); } catch (Exception e1) { logger.warn("failed to write failed status for river creation", e); }