quick update

bin/elasticdump

@@ -8,8 +8,7 @@ const packageData = require(path.join(__dirname, '..', 'package.json'))
 const { isUrl } = require(path.join(__dirname, '..', 'lib', 'is-url.js'))
 const ArgParser = require(path.join(__dirname, '..', 'lib', 'argv.js'))
 const versionCheck = require(path.join(__dirname, '..', 'lib', 'version-check.js'))
-require('aws-sdk/lib/maintenance_mode_message').suppress = true;
-
+require('aws-sdk/lib/maintenance_mode_message').suppress = true
 
 // For future developers.  If you add options here, be sure to add the option to test suite tests where necessary
 const defaults = {
@@ -145,7 +144,7 @@ const defaults = {
   // opensearch compatability
   'force-os-version': '7.10.2',
   // opensearch serverless
-  openSearchServerless: false,
+  openSearchServerless: false
 }
 const options = {}
 

bin/multielasticdump

@@ -20,7 +20,7 @@ const ArgParser = require(path.join(__dirname, '..', 'lib', 'argv.js'))
 const addAuth = require(path.join(__dirname, '..', 'lib', 'add-auth.js'))
 const versionCheck = require(path.join(__dirname, '..', 'lib', 'version-check.js'))
 const AWS = require('aws-sdk')
-require('aws-sdk/lib/maintenance_mode_message').suppress = true;
+require('aws-sdk/lib/maintenance_mode_message').suppress = true
 const initAws = require(path.join(__dirname, '..', 'lib', 'init-aws.js'))
 const requestUtils = require(path.join(__dirname, '..', 'lib', 'request.js'))
 const aws4signer = require(path.join(__dirname, '..', 'lib', 'aws4signer.js'))

lib/aws4signer.js

@@ -73,12 +73,8 @@ const aws4signer = async (esRequest, parent) => {
     esRequest.headers = Object.assign({ host: urlObj.hostname, 'Content-Type': 'application/json' }, esRequest.headers)
     esRequest.path = `${urlObj.pathname}?${urlObj.searchParams.toString()}`
 
-    const sha256 = calculateSHA256(esRequest.body)
-    esRequest.headers = {
-      'X-Amz-Content-SHA256': sha256,
-      host: urlObj.hostname,
-      'Content-Type': 'application/json',
-      ...esRequest.headers
+    if (parent.options.openSearchServerless) {
+      esRequest.headers = Object.assign({ 'X-Amz-Content-SHA256': calculateSHA256(esRequest.body) }, esRequest.headers)
     }
     return aws4.sign(esRequest, credentials)
   }

lib/transports/__es__/_base.js

@@ -35,10 +35,10 @@ class Base {
     }
   }
 
-  version (prefix, isOpenSearchServerless, callback) {
+  version (prefix, callback) {
     if (this.ESversion) { return callback() }
 
-    if (isOpenSearchServerless) {
+    if (this.parent.options.openSearchServerless) {
       this.parent.emit('debug', `opensearch serverless does not support version check, assuming major version: ${this.ESversion}`)
       this.setSearchBody()
       return callback()

lib/transports/__es__/_data.js

@@ -6,79 +6,10 @@ const _ = require('lodash')
 const { scrollResultSet, safeDecodeURIComponent } = require('./_helpers')
 
 class Data {
-  async getOpenSearchServerlessData (limit, offset, callback) {
-    let searchBodyTmp
-
-    if (this.parent.options.searchWithTemplate) {
-      searchBodyTmp = await this.renderTemplate(this.searchBody.id, this.searchBody.params)
-        .then(result => {
-          return result
-        })
-        .catch(error => {
-          throw new Error(error)
-        })
-    } else {
-      searchBodyTmp = this.searchBody
-    }
-
-    const searchBody = searchBodyTmp
-    searchBody.size = limit
-    searchBody.from = offset
-    const additionalParams = this.paramsToString(this.parent.options[`${this.options.type}-params`] || this.parent.options.params, '&')
-    // Note: OpenSearch Serverless does not support _search?scroll API. As a workaround pure _search is used
-    // List of supported endpoints: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html#serverless-operations
-    const uri = `${this.base.url}/_search${additionalParams}`
-
-    const searchRequest = {
-      uri,
-      method: 'GET',
-      body: jsonParser.stringify(searchBody)
-    }
-    aws4signer(searchRequest, this.parent).then((res) => {
-      this.baseRequest(searchRequest, (err, response) => {
-        err = this.handleError(err, response)
-        if (err) {
-          return callback(err, [])
-        }
-
-        const body = jsonParser.parse(response.body, this.parent)
-
-        const hits = _.get(body, 'hits.hits', [])
-        const hitsTotal = _.get(body, 'hits.total.value', body.hits.total)
-        this.totalSearchResults = hitsTotal
-
-        if (hits.length < limit) {
-          // No more data to fetch
-          callback(null, hits)
-        } else {
-          this.getOpenSearchServerlessData(limit, offset + limit, (err, moreHits) => {
-            if (err) {
-              return callback(err, [])
-            }
-            callback(null, hits.concat(moreHits))
-          })
-        }
-      })
-    }).catch(callback)
-  }
-
   async getData (limit, offset, callback) {
     let searchRequest, uri
-    let searchBodyTmp
-
-    if (this.parent.options.searchWithTemplate) {
-      searchBodyTmp = await this.renderTemplate(this.searchBody.id, this.searchBody.params)
-        .then(result => {
-          return result
-        })
-        .catch(error => {
-          throw new Error(error)
-        })
-    } else {
-      searchBodyTmp = this.searchBody
-    }
 
-    const searchBody = searchBodyTmp
+    const searchBody = await this.searchWithTemplate(this.searchBody)
 
     if (offset >= this.totalSearchResults && this.totalSearchResults !== 0) {
       callback(null, [])

lib/transports/__es__/_serverless.js

@@ -0,0 +1,46 @@
+const jsonParser = require('../../jsonparser.js')
+const aws4signer = require('../../aws4signer')
+const _ = require('lodash')
+
+class Serverless {
+  async getOpenSearchServerlessData (limit, offset, callback) {
+    const searchBody = await this.searchWithTemplate(this.searchBody)
+    searchBody.size = limit
+    searchBody.from = offset
+    const additionalParams = this.paramsToString(this.parent.options[`${this.options.type}-params`] || this.parent.options.params, '&')
+    // Note: OpenSearch Serverless does not support _search?scroll API. As a workaround pure _search is used
+    // List of supported endpoints: https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-genref.html#serverless-operations
+    const uri = `${this.base.url}/_search${additionalParams}`
+
+    const searchRequest = {
+      uri,
+      method: 'GET',
+      body: jsonParser.stringify(searchBody)
+    }
+    aws4signer(searchRequest, this.parent).then((res) => {
+      this.baseRequest(searchRequest, (err, response) => {
+        err = this.handleError(err, response)
+        if (err) {
+          return callback(err, [])
+        }
+
+        const body = jsonParser.parse(response.body, this.parent)
+        const hits = _.get(body, 'hits.hits', [])
+
+        if (hits.length < limit) {
+          // No more data to fetch
+          callback(null, hits)
+        } else {
+          this.getOpenSearchServerlessData(limit, offset + limit, (err, moreHits) => {
+            if (err) {
+              return callback(err, [])
+            }
+            callback(null, hits.concat(moreHits))
+          })
+        }
+      })
+    }).catch(callback)
+  }
+}
+
+module.exports = Serverless

lib/transports/__es__/_template.js

@@ -65,6 +65,16 @@ class Template {
     )
   }
 
+  searchWithTemplate (searchBody) {
+    return new Promise((resolve, reject) => {
+      if (this.parent.options.searchWithTemplate) {
+        this.renderTemplate(searchBody.id, searchBody.params).then(resolve).catch(reject)
+      } else {
+        resolve(searchBody)
+      }
+    })
+  }
+
   _transformTemplates (data, type) {
     return _.reduce(data[`${type}s`], (obj, tmpl) => {
       obj[tmpl.name] = tmpl[type]

lib/transports/__es__/index.js

@@ -7,5 +7,6 @@ module.exports = {
   Setting: require('./_setting'),
   Template: require('./_template'),
   Script: require('./_script'),
-  Data: require('./_data')
+  Data: require('./_data'),
+  Serverless: require('./_serverless')
 }

lib/transports/elasticsearch.js

@@ -14,10 +14,11 @@ const {
   Policy,
   Setting,
   Template,
+  Serverless,
   Script
 } = require('./__es__')
 
-class elasticsearch extends Many(Base, Alias, Analyzer, Mapping, Policy, Setting, Template, Script, Data) {
+class elasticsearch extends Many(Base, Alias, Analyzer, Mapping, Policy, Setting, Template, Script, Data, Serverless) {
   constructor (parent, url, options) {
     super()
     this.base = parseBaseURL(url, options)
@@ -80,7 +81,7 @@ class elasticsearch extends Many(Base, Alias, Analyzer, Mapping, Policy, Setting
   // return (error, arr) where arr is an array of objects
   get (limit, offset, callback) {
     const type = this.parent.options.type
-    this.version('input', this.parent.options.openSearchServerless, err => {
+    this.version('input', err => {
       if (err) { return callback(err) }
 
       if (type === 'data') {
@@ -127,7 +128,7 @@ class elasticsearch extends Many(Base, Alias, Analyzer, Mapping, Policy, Setting
   // return (error, writes)
   set (data, limit, offset, callback) {
     const type = this.parent.options.type
-    this.version('output', this.parent.options.openSearchServerless, err => {
+    this.version('output', err => {
       if (err) { return callback(err) }
       if (type === 'data') {
         this.setData(data, limit, offset, callback)

package.json

@@ -32,7 +32,6 @@
     "aws4": "^1.12.0",
     "big.js": "^5.2.2",
     "bytes": "^3.1.2",
-    "crypto": "^1.0.1",
     "delay": "^5.0.0",
     "extends-classes": "1.0.5",
     "fast-csv": "4.3.6",