test: Updated elasticsearch container to ignore highwatermark to func…

…tion on a constrained environment (#1803)

docker-compose.yml

@@ -0,0 +1,107 @@
+version: "3"
+services:
+  elasticsearch:
+    container_name: nr_node_elastic
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.7.1
+    environment:
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      # Set cluster to single node
+      - "discovery.type=single-node"
+      # Disable high watermarks, used in CI as the runner is constrained on disk space
+      - "cluster.routing.allocation.disk.threshold_enabled=false"
+      - "network.host=_site_"
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      # Disable password
+      - "xpack.security.enabled=false"
+    ports:
+      - "9200:9200"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://127.0.0.1:9200"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+  memcached:
+    container_name: nr_node_memcached
+    image: memcached
+    ports:
+      - "11211:11211"
+  mongodb_3:
+    container_name: nr_node_mongodb
+    platform: linux/amd64
+    image: library/mongo:3
+    ports:
+      - "27017:27017"
+    healthcheck:
+      test: ["CMD", "mongo", "--quiet"]
+      interval: 1s
+      timeout: 10s
+      retries: 30
+  mongodb_5:
+    container_name: nr_node_mongodb_5
+    image: library/mongo:5
+    ports:
+      - "27018:27017"
+    healthcheck:
+      test: ["CMD", "mongo", "--quiet"]
+      interval: 1s
+      timeout: 10s
+      retries: 30
+  mysql:
+    container_name: nr_node_mysql
+    platform: linux/amd64
+    image: mysql:5
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_ALLOW_EMPTY_PASSWORD: 1
+    healthcheck:
+      test: ["CMD", "mysql" ,"-h", "mysql", "-P", "3306", "-u", "root", "-e", "SELECT 1"]
+      interval: 1s
+      timeout: 10s
+      retries: 30
+  redis:
+    container_name: nr_node_redis
+    image: redis
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 1s
+      timeout: 10s
+      retries: 30
+  cassandra:
+    container_name: nr_node_cassandra
+    platform: linux/amd64
+    image: zmarcantel/cassandra
+    ports:
+      - "9042:9042"
+    healthcheck:
+      test: [ "CMD", "cqlsh", "-u cassandra", "-p cassandra"]
+      interval: 5s
+      timeout: 10s
+      retries: 6 
+  # pg 9.2 has built in healthcheck
+  pg:
+    container_name: nr_node_postgres
+    image: postgres:9.2
+    ports:
+      - "5432:5432"
+  pg_prisma:
+    container_name: nr_node_postgres_prisma
+    image: postgres:15
+    ports:
+      - "5434:5434"
+    environment:
+      PGPORT: 5434
+      POSTGRES_PASSWORD: postgres
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 1s
+      timeout: 10s
+      retries: 30
+  rmq:
+    container_name: nr_node_rabbit
+    image: rabbitmq:3
+    ports:
+      - "5672:5672"

package.json

@@ -153,7 +153,7 @@
     "lint:lockfile": "lockfile-lint --path package-lock.json --type npm --allowed-hosts npm --validate-https --validate-integrity",
     "public-docs": "jsdoc -c ./jsdoc-conf.json --tutorials examples/shim api.js lib/shim/ lib/transaction/handle.js && cp examples/shim/*.png out/",
     "publish-docs": "./bin/publish-docs.sh",
-    "services": "./bin/docker-services.sh",
+    "services": "docker compose up -d --wait",
     "smoke": "npm run ssl && time tap test/smoke/**/**/*.tap.js --timeout=180 --no-coverage",
     "ssl": "./bin/ssl.sh",
     "sub-install": "node test/bin/install_sub_deps",

test/lib/params.js

@@ -28,8 +28,6 @@ module.exports = {
 
   elastic_host: process.env.NR_NODE_TEST_ELASTIC_HOST || 'localhost',
   elastic_port: process.env.NR_NODE_TEST_ELASTIC_PORT || 9200,
-  elastic_user: process.env.NR_NODE_TEST_ELASTIC_USER || 'elastic',
-  elastic_pass: process.env.NR_NODE_TEST_ELASTIC_PASS || 'changeme',
 
   postgres_host: process.env.NR_NODE_TEST_POSTGRES_HOST || 'localhost',
   postgres_port: process.env.NR_NODE_TEST_POSTGRES_PORT || 5432,

test/versioned/elastic/elasticsearch.tap.js

@@ -10,19 +10,24 @@ const test = tap.test
 const helper = require('../../lib/agent_helper')
 const params = require('../../lib/params')
 const urltils = require('../../../lib/util/urltils')
+const crypto = require('crypto')
+const DB_INDEX = `test-${randomString()}`
+const DB_INDEX_2 = `test2-${randomString()}`
 
-test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
+function randomString() {
+  return crypto.randomBytes(5).toString('hex')
+}
+
+test('Elasticsearch instrumentation', (t) => {
   t.autoend()
 
   let METRIC_HOST_NAME = null
   let HOST_ID = null
-  const DB_INDEX = `test`
-  const DB_INDEX_2 = `test2`
 
   let agent
   let client
 
-  t.beforeEach(async () => {
+  t.before(async () => {
     agent = helper.instrumentMockedAgent()
 
     METRIC_HOST_NAME = urltils.isLocalhost(params.elastic_host)
@@ -33,37 +38,43 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
     // need to capture attributes
     agent.config.attributes.enabled = true
 
-    const { Client, HttpConnection } = require('@elastic/elasticsearch')
+    const { Client } = require('@elastic/elasticsearch')
     client = new Client({
-      node: `http://${params.elastic_host}:${params.elastic_port}`,
-      auth: {
-        username: params.elastic_user,
-        password: params.elastic_pass
-      },
-      Connection: HttpConnection
+      node: `http://${params.elastic_host}:${params.elastic_port}`
     })
+
+    return Promise.all([
+      client.indices.create({ index: DB_INDEX }),
+      client.indices.create({ index: DB_INDEX_2 })
+    ])
   })
 
-  t.afterEach(async () => {
-    agent && helper.unloadAgent(agent)
+  t.afterEach(() => {
+    agent.queries.clear()
   })
 
-  t.teardown(async () => {
-    await client.indices.delete({ index: DB_INDEX })
-    await client.indices.delete({ index: DB_INDEX_2 })
+  t.teardown(() => {
+    agent && helper.unloadAgent(agent)
+    return Promise.all([
+      client.indices.delete({ index: DB_INDEX }),
+      client.indices.delete({ index: DB_INDEX_2 })
+    ])
   })
 
   t.test('should be able to record creating an index', async (t) => {
+    const index = `test-index-${randomString()}`
+    t.teardown(async () => {
+      await client.indices.delete({ index })
+    })
     await helper.runInTransaction(agent, async function transactionInScope(transaction) {
       t.ok(transaction, 'transaction should be visible')
-      await client.indices.create({ index: DB_INDEX })
-      await client.indices.create({ index: DB_INDEX_2 })
+      await client.indices.create({ index })
       const trace = transaction.trace
       t.ok(trace?.root?.children?.[0], 'trace, trace root, and first child should exist')
       const firstChild = trace.root.children[0]
       t.equal(
         firstChild.name,
-        'Datastore/statement/ElasticSearch/test/index.create',
+        `Datastore/statement/ElasticSearch/${index}/index.create`,
         'should record index PUT as create'
       )
     })
@@ -111,22 +122,21 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
     agent.config.slow_sql.enabled = true
     await helper.runInTransaction(agent, async function transactionInScope(transaction) {
       const expectedQuery = { q: 'sixth' }
-      const search = await client.search({ index: DB_INDEX_2, q: 'sixth' })
+      const search = await client.search({ index: DB_INDEX_2, ...expectedQuery })
       t.ok(search, 'search should return a result')
       t.ok(transaction, 'transaction should still be visible after search')
       const trace = transaction.trace
       t.ok(trace?.root?.children?.[0], 'trace, trace root, and first child should exist')
       const firstChild = trace.root.children[0]
       t.match(
         firstChild.name,
-        'Datastore/statement/ElasticSearch/test2/search',
+        `Datastore/statement/ElasticSearch/${DB_INDEX_2}/search`,
         'querystring search should be recorded as a search'
       )
       const attrs = firstChild.getAttributes()
       t.match(attrs.product, 'ElasticSearch')
       t.match(attrs.host, METRIC_HOST_NAME)
       transaction.end()
-      // can we inspect recorded query?
       t.ok(agent.queries.samples.size > 0, 'there should be a query sample')
       for (const query of agent.queries.samples.values()) {
         t.ok(query.total > 0, 'the samples should have positive duration')
@@ -153,14 +163,16 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
       const firstChild = trace.root.children[0]
       t.match(
         firstChild.name,
-        'Datastore/statement/ElasticSearch/test/search',
+        `Datastore/statement/ElasticSearch/${DB_INDEX}/search`,
         'search index is specified, so name shows it'
       )
       const attrs = firstChild.getAttributes()
-      t.match(attrs.product, 'ElasticSearch')
-      t.match(attrs.host, METRIC_HOST_NAME)
+      t.equal(attrs.product, 'ElasticSearch')
+      t.equal(attrs.host, METRIC_HOST_NAME)
+      t.equal(attrs.port_path_or_id, `${params.elastic_port}`)
+      // TODO: update once instrumentation is properly setting database name
+      t.equal(attrs.database_name, 'unknown')
       transaction.end()
-      // can we inspect recorded query?
       t.ok(agent.queries.samples.size > 0, 'there should be a query sample')
       for (const query of agent.queries.samples.values()) {
         t.ok(query.total > 0, 'the samples should have positive duration')
@@ -195,7 +207,6 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
       t.match(attrs.product, 'ElasticSearch')
       t.match(attrs.host, METRIC_HOST_NAME)
       transaction.end()
-      // can we inspect recorded query?
       t.ok(agent.queries.samples.size > 0, 'there should be a query sample')
       for (const query of agent.queries.samples.values()) {
         t.ok(query.total > 0, 'the samples should have positive duration')
@@ -260,22 +271,23 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
   })
 
   t.test('should create correct metrics', async function (t) {
+    const id = `key-${randomString()}`
     t.plan(28)
     await helper.runInTransaction(agent, async function transactionInScope(transaction) {
       await client.index({
         index: DB_INDEX,
-        id: 'testkey2',
+        id,
         document: {
           title: 'second document',
           body: 'body of the second document'
         }
       })
 
       // check metrics/methods for "exists" queries
-      await client.exists({ id: 'testkey2', index: DB_INDEX })
-      await client.get({ id: 'testkey2', index: DB_INDEX })
+      await client.exists({ id, index: DB_INDEX })
+      await client.get({ id, index: DB_INDEX })
       await client.search({ query: { match: { body: 'document' } } })
-      await client.delete({ id: 'testkey2', index: DB_INDEX })
+      await client.delete({ id, index: DB_INDEX })
       transaction.end()
 
       const unscoped = transaction.metrics.unscoped
@@ -288,10 +300,10 @@ test('Elasticsearch instrumentation', { timeout: 20000 }, (t) => {
         'Datastore/operation/ElasticSearch/doc.get': 1,
         'Datastore/operation/ElasticSearch/doc.exists': 1,
         'Datastore/operation/ElasticSearch/search': 1,
-        'Datastore/statement/ElasticSearch/test/doc.create': 1,
-        'Datastore/statement/ElasticSearch/test/doc.get': 1,
-        'Datastore/statement/ElasticSearch/test/doc.exists': 1,
-        'Datastore/statement/ElasticSearch/test/doc.delete': 1,
+        [`Datastore/statement/ElasticSearch/${DB_INDEX}/doc.create`]: 1,
+        [`Datastore/statement/ElasticSearch/${DB_INDEX}/doc.get`]: 1,
+        [`Datastore/statement/ElasticSearch/${DB_INDEX}/doc.exists`]: 1,
+        [`Datastore/statement/ElasticSearch/${DB_INDEX}/doc.delete`]: 1,
         'Datastore/statement/ElasticSearch/any/search': 1
       }
       expected['Datastore/instance/ElasticSearch/' + HOST_ID] = 5