Merge pull request #4964 from kdembler/argus-data-streams

[Argus] use ES data streams, enable log correlation
Joystream · Nov 17, 2023 · 8212966 · 8212966
2 parents 1289bab + 5499c01
commit 8212966
Show file tree

Hide file tree

Showing 15 changed files with 144 additions and 72 deletions.
diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md
@@ -1,6 +1,10 @@
+## 1.5.0
+
+- Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name.
+
 ## 1.4.1
 
-- Bumped `winston-elasticsearch` package verion 
+- Bumped `winston-elasticsearch` package verion
 - **FIX**: Added error handler to caught exception in `ElasticsearchTransport` and gracefully log them
 
 ### 1.4.0

diff --git a/distributor-node/config.yml b/distributor-node/config.yml
@@ -20,17 +20,17 @@ logs:
   #     username: username
   #     password: password
 limits:
-  storage: 100G
+  storage: 500G
   maxConcurrentStorageNodeDownloads: 100
   maxConcurrentOutboundConnections: 300
   outboundRequestsTimeoutMs: 5000
   pendingDownloadTimeoutSec: 3600
-  maxCachedItemSize: 1G
+  maxCachedItemSize: 20G
   queryNodeCacheTTL: 60
 intervals:
   saveCacheState: 60
   checkStorageNodeResponseTimes: 60
-  cacheCleanup: 7200 # every 2h, cache cleanup currently is very heavy on QN
+  cacheCleanup: 86400 # every 24h, cache cleanup currently is very heavy on QN
 publicApi:
   port: 3334
 operatorApi:

diff --git a/...on-properties-logs-properties-elasticsearch-logging-options-properties-index.md b/...on-properties-logs-properties-elasticsearch-logging-options-properties-index.md
diff --git a/...perties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md b/...perties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md
@@ -0,0 +1,3 @@
+## indexPrefix Type
+
+`string`
diff --git a/...s/schema/definition-properties-logs-properties-elasticsearch-logging-options.md b/...s/schema/definition-properties-logs-properties-elasticsearch-logging-options.md
@@ -4,12 +4,12 @@
 
 # elastic Properties
 
-| Property              | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                        |
-| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [level](#level)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                |
-| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") |
-| [index](#index)       | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/index")       |
-| [auth](#auth)         | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth")         |
+| Property                    | Type     | Required | Nullable       | Defined by                                                                                                                                                                                                                              |
+| :-------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [level](#level)             | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level")                      |
+| [endpoint](#endpoint)       | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint")       |
+| [indexPrefix](#indexprefix) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/indexPrefix") |
+| [auth](#auth)               | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth")               |
 
 ## level
 
@@ -65,21 +65,21 @@ Elastichsearch endpoint to push the logs to (for example: <http://localhost:9200
 
 **URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")
 
-## index
+## indexPrefix
 
-Elasticsearch index to push the logs to. If not provided, will fallback to "distributor-node"
+Elasticsearch data stream prefix to push the logs to. `-${config.id}` will be automatically appended. If not provided, will fallback to "logs-argus"
 
-`index`
+`indexPrefix`
 
 *   is optional
 
 *   Type: `string`
 
 *   cannot be null
 
-*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/index")
+*   defined in: [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/indexPrefix")
 
-### index Type
+### indexPrefix Type
 
 `string`
 

diff --git a/distributor-node/package.json b/distributor-node/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"
@@ -161,7 +161,7 @@
     "checks": "tsc --noEmit --pretty && prettier ./ --check && yarn lint",
     "cli": "./bin/run",
     "start": "./bin/run start",
-    "start:with-instrumentation": "export OTEL_APPLICATION=distributor-node; node --require @joystream/opentelemetry ./bin/run start"
+    "start:with-instrumentation": "export OTEL_APPLICATION=distributor-node; export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200; export OTEL_RESOURCE_ATTRIBUTES=service.name=argus,deployment.environment=production; node --require @joystream/opentelemetry ./bin/run start"
   },
   "types": "lib/index.d.ts"
 }
diff --git a/distributor-node/src/schemas/configSchema.ts b/distributor-node/src/schemas/configSchema.ts
@@ -105,9 +105,10 @@ export const configSchema: JSONSchema4 = objectSchema({
               type: 'string',
               format: 'uri',
             },
-            index: {
+            indexPrefix: {
               description:
-                'Elasticsearch index to push the logs to. If not provided, will fallback to "distributor-node"',
+                // eslint-disable-next-line no-template-curly-in-string
+                'Elasticsearch data stream prefix to push the logs to. `-${config.id}` will be automatically appended. If not provided, will fallback to "logs-argus"',
               type: 'string',
             },
             auth: objectSchema({

diff --git a/distributor-node/src/services/logging/LoggingService.ts b/distributor-node/src/services/logging/LoggingService.ts
@@ -85,12 +85,40 @@ export class LoggingService {
 
     let esTransport: ElasticsearchTransport | undefined
     if (config.logs?.elastic) {
+      const indexPrefix = config.logs.elastic.indexPrefix || 'logs-argus'
+      const index = `${indexPrefix}-${config.id}`.toLowerCase()
       esTransport = new ElasticsearchTransport({
-        index: config.logs.elastic.index || 'distributor-node',
+        index,
+        dataStream: true,
         level: config.logs.elastic.level,
         format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()),
         retryLimit: 10,
         flushInterval: 5000,
+        // apply custom transform so that tracing data (if present) is placed in the top level of the log
+        // based on https://github.com/vanthome/winston-elasticsearch/blob/d948fa1b705269a4713480593ea657de34c0a942/transformer.js
+        transformer: (logData) => {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const transformed: any = {}
+          transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString()
+          transformed.message = logData.message
+          transformed.severity = logData.level
+          transformed.fields = logData.meta
+
+          if (logData.meta.trace_id || logData.meta.trace_flags) {
+            transformed.trace = {
+              id: logData.meta.trace_id,
+              flags: logData.meta.trace_flags,
+            }
+          }
+          if (logData.meta.span_id) {
+            transformed.span = { id: logData.meta.span_id }
+          }
+          if (logData.meta.transaction_id) {
+            transformed.transaction = { id: logData.meta.transaction_id }
+          }
+
+          return transformed
+        },
         source: config.id,
         clientOpts: {
           node: {

diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml
@@ -19,7 +19,7 @@ services:
     volumes:
       - es-data:/usr/share/elasticsearch/data
     ports:
-      - 127.0.0.1:9200:9200
+      - 9200:9200
     networks:
       - joystream_default
 

diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md
@@ -1,9 +1,12 @@
+### 3.8.0
+
+- Changed Elasticsearch transport to use data streams instead of regular indices. Removed `--elasticSearchIndex` option and replaced with `--elasticSearchIndexPrefix`. Node ID from config will be automatically appended to the index name.
+
 ### 3.7.2
 
-- Bumped `winston-elasticsearch` package verion 
+- Bumped `winston-elasticsearch` package verion
 - **FIX**: Added error handler to caught exception in `ElasticsearchTransport` and gracefully log them
 
-
 ### 3.7.1
 
 - Disable open-api express response validation if NODE_ENV == 'production'. This should improve response times when serving assets.

diff --git a/storage-node/README.md b/storage-node/README.md
@@ -6,17 +6,18 @@ Joystream storage subsystem.
 ![License](https://img.shields.io/github/license/Joystream/joystream)
 
 <!-- toc -->
-* [Colossus v2](#colossus-v2)
-* [Description](#description)
-* [Installation](#installation)
-* [Ubuntu Linux](#ubuntu-linux)
-* [Install packages required for installation](#install-packages-required-for-installation)
-* [Clone the code repository](#clone-the-code-repository)
-* [Install volta](#install-volta)
-* [Install project dependencies and build it](#install-project-dependencies-and-build-it)
-* [Verify installation](#verify-installation)
-* [Usage](#usage)
-* [CLI Commands](#cli-commands)
+
+- [Colossus v2](#colossus-v2)
+- [Description](#description)
+- [Installation](#installation)
+- [Ubuntu Linux](#ubuntu-linux)
+- [Install packages required for installation](#install-packages-required-for-installation)
+- [Clone the code repository](#clone-the-code-repository)
+- [Install volta](#install-volta)
+- [Install project dependencies and build it](#install-project-dependencies-and-build-it)
+- [Verify installation](#verify-installation)
+- [Usage](#usage)
+- [CLI Commands](#cli-commands)
 <!-- tocstop -->
 
 # Description
@@ -147,28 +148,29 @@ There is also an option to run Colossus as [Docker container](../colossus.Docker
 # CLI Commands
 
 <!-- commands -->
-* [`storage-node dev:multihash`](#storage-node-devmultihash)
-* [`storage-node dev:sync`](#storage-node-devsync)
-* [`storage-node dev:verify-bag-id`](#storage-node-devverify-bag-id)
-* [`storage-node help [COMMAND]`](#storage-node-help-command)
-* [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite)
-* [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket)
-* [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket)
-* [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator)
-* [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator)
-* [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits)
-* [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status)
-* [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit)
-* [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags)
-* [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist)
-* [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status)
-* [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee)
-* [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond)
-* [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy)
-* [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits)
-* [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation)
-* [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata)
-* [`storage-node server`](#storage-node-server)
+
+- [`storage-node dev:multihash`](#storage-node-devmultihash)
+- [`storage-node dev:sync`](#storage-node-devsync)
+- [`storage-node dev:verify-bag-id`](#storage-node-devverify-bag-id)
+- [`storage-node help [COMMAND]`](#storage-node-help-command)
+- [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite)
+- [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket)
+- [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket)
+- [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator)
+- [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator)
+- [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits)
+- [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status)
+- [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit)
+- [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags)
+- [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist)
+- [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status)
+- [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee)
+- [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond)
+- [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy)
+- [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits)
+- [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation)
+- [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata)
+- [`storage-node server`](#storage-node-server)
 
 ## `storage-node dev:multihash`
 
@@ -845,7 +847,7 @@ OPTIONS
 
   -z, --logFileChangeFrequency=(yearly|monthly|daily|hourly|none)  [default: daily] Log files update frequency.
 
-  --elasticSearchIndex=elasticSearchIndex                          Elasticsearch index name.
+  --elasticSearchIndexPrefix=elasticSearchIndexPrefix              Elasticsearch index prefix. Node ID will be appended to the prefix. Default: logs-colossus.
 
   --elasticSearchPassword=elasticSearchPassword                    Elasticsearch password for basic authentication.
 
@@ -856,4 +858,5 @@ OPTIONS
 ```
 
 _See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/master/src/commands/server.ts)_
+
 <!-- commandsstop -->
diff --git a/storage-node/package.json b/storage-node/package.json
@@ -1,7 +1,7 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "3.7.2",
+  "version": "3.8.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"
@@ -160,7 +160,7 @@
     "ensure": "yarn format && yarn lint --fix && yarn build",
     "checks": "tsc --noEmit --pretty && prettier ./src --check && yarn lint",
     "start": "./bin/run server",
-    "start:with-instrumentation": "export OTEL_APPLICATION=storage-node; node --require @joystream/opentelemetry ./bin/run server"
+    "start:with-instrumentation": "export OTEL_APPLICATION=storage-node; export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200; export OTEL_RESOURCE_ATTRIBUTES=service.name=colossus,deployment.environment=production; node --require @joystream/opentelemetry ./bin/run server"
   },
   "types": "lib/index.d.ts"
 }
diff --git a/storage-node/src/commands/server.ts b/storage-node/src/commands/server.ts
@@ -88,10 +88,10 @@ export default class Server extends ApiCommandBase {
 Log level could be set using the ELASTIC_LOG_LEVEL enviroment variable.
 Supported values: warn, error, debug, info. Default:debug`,
     }),
-    elasticSearchIndex: flags.string({
+    elasticSearchIndexPrefix: flags.string({
       required: false,
-      env: 'ELASTIC_INDEX',
-      description: 'Elasticsearch index name.',
+      env: 'ELASTIC_INDEX_PREFIX',
+      description: 'Elasticsearch index prefix. Node ID will be appended to the prefix. Default: logs-colossus',
     }),
     elasticSearchUser: flags.string({
       dependsOn: ['elasticSearchEndpoint', 'elasticSearchPassword'],
@@ -139,7 +139,7 @@ Supported values: warn, error, debug, info. Default:debug`,
       initNewLogger({
         elasticSearchlogSource: logSource,
         elasticSearchEndpoint: flags.elasticSearchEndpoint,
-        elasticSearchIndex: flags.elasticSearchIndex,
+        elasticSearchIndexPrefix: flags.elasticSearchIndexPrefix,
         elasticSearchUser: flags.elasticSearchUser,
         elasticSearchPassword: flags.elasticSearchPassword,
         filePath: flags.logFilePath,