Stats: add segments.index_writer_max_memory to see index writer's max…

… RAM usage before buffered documents must be written to a new segment Closes #7438 Closes #7440
elastic · Aug 25, 2014 · 783a9cb · 783a9cb
1 parent 7428786
commit 783a9cb
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 2 deletions.
diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc
@@ -180,7 +180,9 @@ operations |9
 |`segments.memory` |`sm`, `segmentsMemory` |No |Memory used by
 segments |1.4kb
 |`segments.index_writer_memory` |`siwm`, `segmentsIndexWriterMemory` |No
-|Memory used by index writer |1.2kb
+|Memory used by index writer |18mb
+|`segments.index_writer_max_memory` |`siwmx`, `segmentsIndexWriterMaxMemory` |No
+|Maximum memory index writer may use before it must write buffered documents to a new segment |32mb
 |`segments.version_map_memory` |`svmm`, `segmentsVersionMapMemory` |No
 |Memory used by version map |1.0kb
 |=======================================================================
diff --git a/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
@@ -38,6 +38,7 @@ public class SegmentsStats implements Streamable, ToXContent {
     private long count;
     private long memoryInBytes;
     private long indexWriterMemoryInBytes;
+    private long indexWriterMaxMemoryInBytes;
     private long versionMapMemoryInBytes;
 
     public SegmentsStats() {
@@ -53,6 +54,10 @@ public void addIndexWriterMemoryInBytes(long indexWriterMemoryInBytes) {
         this.indexWriterMemoryInBytes += indexWriterMemoryInBytes;
     }
 
+    public void addIndexWriterMaxMemoryInBytes(long indexWriterMaxMemoryInBytes) {
+        this.indexWriterMaxMemoryInBytes += indexWriterMaxMemoryInBytes;
+    }
+
     public void addVersionMapMemoryInBytes(long versionMapMemoryInBytes) {
         this.versionMapMemoryInBytes += versionMapMemoryInBytes;
     }
@@ -63,6 +68,7 @@ public void add(SegmentsStats mergeStats) {
         }
         add(mergeStats.count, mergeStats.memoryInBytes);
         addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
+        addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes);
         addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes);
     }
 
@@ -95,6 +101,17 @@ public ByteSizeValue getIndexWriterMemory() {
         return new ByteSizeValue(indexWriterMemoryInBytes);
     }
 
+    /**
+     * Maximum memory index writer may use before it must write buffered documents to a new segment.
+     */
+    public long getIndexWriterMaxMemoryInBytes() {
+        return this.indexWriterMaxMemoryInBytes;
+    }
+
+    public ByteSizeValue getIndexWriterMaxMemory() {
+        return new ByteSizeValue(indexWriterMaxMemoryInBytes);
+    }
+
     /**
      * Estimation of the memory usage by version map
      */
@@ -118,6 +135,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         builder.field(Fields.COUNT, count);
         builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes);
         builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
+        builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes);
         builder.byteSizeField(Fields.VERSION_MAP_MEMORY_IN_BYTES, Fields.VERSION_MAP_MEMORY, versionMapMemoryInBytes);
         builder.endObject();
         return builder;
@@ -130,6 +148,8 @@ static final class Fields {
         static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
         static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory");
         static final XContentBuilderString INDEX_WRITER_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_memory_in_bytes");
+        static final XContentBuilderString INDEX_WRITER_MAX_MEMORY = new XContentBuilderString("index_writer_max_memory");
+        static final XContentBuilderString INDEX_WRITER_MAX_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_max_memory_in_bytes");
         static final XContentBuilderString VERSION_MAP_MEMORY = new XContentBuilderString("version_map_memory");
         static final XContentBuilderString VERSION_MAP_MEMORY_IN_BYTES = new XContentBuilderString("version_map_memory_in_bytes");
     }
@@ -142,6 +162,9 @@ public void readFrom(StreamInput in) throws IOException {
             indexWriterMemoryInBytes = in.readLong();
             versionMapMemoryInBytes = in.readLong();
         }
+        if (in.getVersion().onOrAfter(Version.V_1_4_0)) {
+            indexWriterMaxMemoryInBytes = in.readLong();
+        }
     }
 
     @Override
@@ -152,5 +175,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeLong(indexWriterMemoryInBytes);
             out.writeLong(versionMapMemoryInBytes);
         }
+        if (out.getVersion().onOrAfter(Version.V_1_4_0)) {
+            out.writeLong(indexWriterMaxMemoryInBytes);
+        }
     }
-}
+}
diff --git a/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java b/src/main/java/org/elasticsearch/index/engine/internal/InternalEngine.java
@@ -1149,6 +1149,7 @@ public SegmentsStats segmentsStats() {
                 }
                 stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed());
                 stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed());
+                stats.addIndexWriterMaxMemoryInBytes((long) (indexWriter.getConfig().getRAMBufferSizeMB()*1024*1024));
                 return stats;
             } finally {
                 searcher.close();

diff --git a/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
@@ -255,6 +255,9 @@ Table getTableWithHeader(final RestRequest request) {
         table.addCell("segments.index_writer_memory", "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
         table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer");
 
+        table.addCell("segments.index_writer_max_memory", "sibling:pri;alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
+        table.addCell("pri.segments.index_writer_max_memory", "default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
+
         table.addCell("segments.version_map_memory", "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
         table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map");
 
@@ -446,6 +449,9 @@ private Table buildTable(RestRequest request, String[] indices, ClusterHealthRes
             table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
 
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMaxMemory());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMaxMemory());
+
             table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());
 

diff --git a/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java b/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
@@ -175,6 +175,7 @@ Table getTableWithHeader(final RestRequest request) {
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
         table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
+        table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
         table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
 
         table.addCell("suggest.current", "alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops");
@@ -284,6 +285,7 @@ private Table buildTable(RestRequest req, ClusterStateResponse state, NodesInfoR
             table.addCell(stats == null ? null : stats.getIndices().getSegments().getCount());
             table.addCell(stats == null ? null : stats.getIndices().getSegments().getMemory());
             table.addCell(stats == null ? null : stats.getIndices().getSegments().getIndexWriterMemory());
+            table.addCell(stats == null ? null : stats.getIndices().getSegments().getIndexWriterMaxMemory());
             table.addCell(stats == null ? null : stats.getIndices().getSegments().getVersionMapMemory());
 
             table.addCell(stats == null ? null : stats.getIndices().getSuggest().getCurrent());

diff --git a/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
@@ -146,6 +146,7 @@ Table getTableWithHeader(final RestRequest request) {
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
         table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
+        table.addCell("segments.index_writer_max_memory", "alias:siwmx,segmentsIndexWriterMaxMemory;default:false;text-align:right;desc:maximum memory index writer may use before it must write buffered documents to a new segment");
         table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
 
         table.addCell("warmer.current", "alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
@@ -245,6 +246,7 @@ private Table buildTable(RestRequest request, ClusterStateResponse state, Indice
             table.addCell(shardStats == null ? null : shardStats.getSegments().getCount());
             table.addCell(shardStats == null ? null : shardStats.getSegments().getMemory());
             table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMemory());
+            table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMaxMemory());
             table.addCell(shardStats == null ? null : shardStats.getSegments().getVersionMapMemory());
 
             table.addCell(shardStats == null ? null : shardStats.getWarmer().current());

diff --git a/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java b/src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java
@@ -434,6 +434,7 @@ public void testSegmentsStats() {
 
         IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
         assertThat(stats.getTotal().getSegments().getIndexWriterMemoryInBytes(), greaterThan(0l));
+        assertThat(stats.getTotal().getSegments().getIndexWriterMaxMemoryInBytes(), greaterThan(0l));
         assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0l));
 
         client().admin().indices().prepareFlush().get();