diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 53f47fc88cdb2..31f4039bcfaca 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -546,23 +546,30 @@ PUT _cluster/settings ---- [discrete] +[[troubleshooting-max-docs-limit]] ==== Number of documents in the shard cannot exceed [2147483519] - -Elasticsearch shards reflect Lucene's underlying https://github.com/apache/lucene/issues/5176[index -`MAX_DOC` hard limit] of 2,147,483,519 (`(2^31)-129`) docs. This figure is -the sum of `docs.count` plus `docs.deleted` as reported by the <> -per shard. Exceeding this limit will result in errors like the following: +Each {es} shard is a separate Lucene index, so it shares Lucene's +https://github.com/apache/lucene/issues/5176[`MAX_DOC` limit] of having at most +2,147,483,519 (`(2^31)-129`) documents. This per-shard limit applies to the sum +of `docs.count` plus `docs.deleted` as reported by the <>. Exceeding this limit will result in errors like the following: [source,txt] ---- Elasticsearch exception [type=illegal_argument_exception, reason=Number of documents in the shard cannot exceed [2147483519]] ---- -TIP: This calculation may differ from the <> calculation, because the Count API does not include nested documents. +TIP: This calculation may differ from the <> +calculation, because the Count API does not include nested documents and does +not count deleted documents. +This limit is much higher than the <> of approximately 200M documents per shard. -Try using the <> to clear deleted docs. For example: +If you encounter this problem, try to mitigate it by using the +<> to merge away some deleted docs. For +example: [source,console] ---- @@ -570,10 +577,9 @@ POST my-index-000001/_forcemerge?only_expunge_deletes=true ---- // TEST[setup:my_index] -This will launch an asynchronous task which can be monitored via the <>. - -For a long-term solution try: +This will launch an asynchronous task which can be monitored via the +<>. -* <> -* aligning the index to recommendations on this page by either -<> or <> the index +It may also be helpful to <>, +or to <> or <> the index into +one with a larger number of shards. diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index 770ed4d213c55..65225530b7d38 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -77,6 +77,7 @@ public enum ReferenceDocs { NETWORK_BINDING_AND_PUBLISHING, SNAPSHOT_REPOSITORY_ANALYSIS, S3_COMPATIBLE_REPOSITORIES, + LUCENE_MAX_DOCS_LIMIT, // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner ; diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 03d244cd8e4ef..8b1602e2ea2ca 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -45,6 +45,8 @@ import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.service.ClusterApplierService; +import org.elasticsearch.common.ReferenceDocs; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.LoggerInfoStream; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; @@ -1688,7 +1690,13 @@ private Exception tryAcquireInFlightDocs(Operation operation, int addingDocs) { final long totalDocs = indexWriter.getPendingNumDocs() + inFlightDocCount.addAndGet(addingDocs); if (totalDocs > maxDocs) { releaseInFlightDocs(addingDocs); - return new IllegalArgumentException("Number of documents in the shard cannot exceed [" + maxDocs + "]"); + return new IllegalArgumentException( + Strings.format( + "Number of documents in the shard cannot exceed [%d]; for more information, see [%s]", + maxDocs, + ReferenceDocs.LUCENE_MAX_DOCS_LIMIT + ) + ); } else { return null; } diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index febcaec1ba057..cd325ce9010ac 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -37,5 +37,6 @@ "ALLOCATION_EXPLAIN_API": "cluster-allocation-explain.html", "NETWORK_BINDING_AND_PUBLISHING": "modules-network.html#modules-network-binding-publishing", "SNAPSHOT_REPOSITORY_ANALYSIS": "repo-analysis-api.html", - "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services" + "S3_COMPATIBLE_REPOSITORIES": "repository-s3.html#repository-s3-compatible-services", + "LUCENE_MAX_DOCS_LIMIT": "size-your-shards.html#troubleshooting-max-docs-limit" }