From f139a1caa7fe998fefdbc200356459d0b1e109e1 Mon Sep 17 00:00:00 2001
From: Alban Mouton <alban.mouton@gmail.com>
Date: Tue, 28 Apr 2020 14:45:56 +0200
Subject: [PATCH] fix: memory usage of the preserve-extension stream

---
 server/utils/extensions.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/server/utils/extensions.js b/server/utils/extensions.js
index 727579fcb1..09533d9f4b 100644
--- a/server/utils/extensions.js
+++ b/server/utils/extensions.js
@@ -391,6 +391,7 @@ class PreserveExtensionStream extends Transform {
     this.mappings = {}
     this.extensionsMap = {}
     this.buffer = []
+    this.bufferChars = 0
 
     // The ES index was not yet created, we will not try to extract previous extensions
     if (!await esClient.indices.exists({ index: this.indexName })) return
@@ -412,6 +413,7 @@ class PreserveExtensionStream extends Transform {
     const searches = []
     const items = this.buffer
     this.buffer = []
+    this.bufferChars = 0
     items.forEach(item => {
       Object.keys(this.mappings).forEach(extensionKey => {
         const [mappedItem, h] = this.mappings[extensionKey]({ doc: item })
@@ -447,7 +449,12 @@ class PreserveExtensionStream extends Transform {
     try {
       if (!this.mappings) await this.init()
       this.buffer.push(item)
-      if (this.i % 1000 === 0) {
+      this.bufferChars += JSON.stringify(item).length
+      if (
+        this.buffer.length >= config.elasticsearch.maxBulkLines ||
+        this.bufferChars >= config.elasticsearch.maxBulkChars
+      ) {
+        debug(`preserve extension stream send buffer of ${this.buffer.length} items totalling ${this.bufferChars} chars`)
         await this._sendBuffer()
       }
       callback()