From b514e75de0b384e7e307414b32fc5d007dcb0f06 Mon Sep 17 00:00:00 2001 From: Klaudiusz Dembler Date: Sun, 12 Nov 2023 17:11:00 +0100 Subject: [PATCH 1/4] [Argus] use ES data streams, enable log correlation --- distributor-node/CHANGELOG.md | 7 ++++- distributor-node/config.yml | 6 ++-- ...search-logging-options-properties-index.md | 3 -- ...-logging-options-properties-indexprefix.md | 3 ++ ...roperties-elasticsearch-logging-options.md | 22 +++++++------- distributor-node/package.json | 4 +-- distributor-node/src/schemas/configSchema.ts | 5 ++-- .../src/services/logging/LoggingService.ts | 29 ++++++++++++++++++- docker-compose.elasticsearch.yml | 2 +- 9 files changed, 57 insertions(+), 24 deletions(-) delete mode 100644 distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md create mode 100644 distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md index 30ce6dd632..d30e599bf5 100644 --- a/distributor-node/CHANGELOG.md +++ b/distributor-node/CHANGELOG.md @@ -1,6 +1,11 @@ +## 1.5.0 + +- Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name. +- Updated example distributor node config to match values currently used in production. + ## 1.4.1 -- Bumped `winston-elasticsearch` package verion +- Bumped `winston-elasticsearch` package verion - **FIX**: Added error handler to caught exception in `ElasticsearchTransport` and gracefully log them ### 1.4.0 diff --git a/distributor-node/config.yml b/distributor-node/config.yml index 739e27dd8b..9883939d37 100644 --- a/distributor-node/config.yml +++ b/distributor-node/config.yml @@ -20,17 +20,17 @@ logs: # username: username # password: password limits: - storage: 100G + storage: 500G maxConcurrentStorageNodeDownloads: 100 maxConcurrentOutboundConnections: 300 outboundRequestsTimeoutMs: 5000 pendingDownloadTimeoutSec: 3600 - maxCachedItemSize: 1G + maxCachedItemSize: 20G queryNodeCacheTTL: 60 intervals: saveCacheState: 60 checkStorageNodeResponseTimes: 60 - cacheCleanup: 7200 # every 2h, cache cleanup currently is very heavy on QN + cacheCleanup: 86400 # every 24h, cache cleanup currently is very heavy on QN publicApi: port: 3334 operatorApi: diff --git a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md deleted file mode 100644 index 259a6110d3..0000000000 --- a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md +++ /dev/null @@ -1,3 +0,0 @@ -## index Type - -`string` diff --git a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md new file mode 100644 index 0000000000..251b2a7c8d --- /dev/null +++ b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md @@ -0,0 +1,3 @@ +## indexPrefix Type + +`string` diff --git a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md index 4cd4948674..49cdf081cb 100644 --- a/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md +++ b/distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md @@ -4,12 +4,12 @@ # elastic Properties -| Property | Type | Required | Nullable | Defined by | -| :-------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [level](#level) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level") | -| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") | -| [index](#index) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-index.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/index") | -| [auth](#auth) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth") | +| Property | Type | Required | Nullable | Defined by | +| :-------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [level](#level) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-file-logging-options-properties-level.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/level") | +| [endpoint](#endpoint) | `string` | Required | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/endpoint") | +| [indexPrefix](#indexprefix) | `string` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-indexprefix.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/indexPrefix") | +| [auth](#auth) | `object` | Optional | cannot be null | [Distributor node configuration](definition-properties-logs-properties-elasticsearch-logging-options-properties-auth.md "https://joystream.org/schemas/argus/config#/properties/logs/properties/elastic/properties/auth") | ## level @@ -65,11 +65,11 @@ Elastichsearch endpoint to push the logs to (for example: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const transformed: any = {} + transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString() + transformed.message = logData.message + transformed.severity = logData.level + transformed.fields = logData.meta + + if (logData.meta.trace_id || logData.meta.trace_flags) { + transformed.trace = { + id: logData.meta.trace_id, + flags: logData.meta.trace_flags, + } + } + if (logData.meta.span_id) { + transformed.span = { id: logData.meta.span_id } + } + if (logData.meta.transaction_id) { + transformed.transaction = { id: logData.meta.transaction_id } + } + + return transformed + }, source: config.id, clientOpts: { node: { diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml index db28b73c2e..5e2a6ea762 100644 --- a/docker-compose.elasticsearch.yml +++ b/docker-compose.elasticsearch.yml @@ -19,7 +19,7 @@ services: volumes: - es-data:/usr/share/elasticsearch/data ports: - - 127.0.0.1:9200:9200 + - 9200:9200 networks: - joystream_default From 9a1922f335c4ebf4adcf8a0607544dd4c2dde82d Mon Sep 17 00:00:00 2001 From: Klaudiusz Dembler Date: Mon, 13 Nov 2023 12:13:53 +0100 Subject: [PATCH 2/4] [Colossus] use ES data streams --- distributor-node/CHANGELOG.md | 1 - .../src/services/logging/LoggingService.ts | 3 +- storage-node/CHANGELOG.md | 7 +- storage-node/README.md | 71 ++++++++++--------- storage-node/package.json | 4 +- storage-node/src/commands/server.ts | 8 +-- storage-node/src/services/logger.ts | 37 ++++++++-- 7 files changed, 83 insertions(+), 48 deletions(-) diff --git a/distributor-node/CHANGELOG.md b/distributor-node/CHANGELOG.md index d30e599bf5..08fef3b1f8 100644 --- a/distributor-node/CHANGELOG.md +++ b/distributor-node/CHANGELOG.md @@ -1,7 +1,6 @@ ## 1.5.0 - Changed Elasticsearch transport to use data streams instead of regular indices. Renamed `config.logs.elastic.index` to `config.logs.elastic.indexPrefix`. Node ID from config will be automatically appended to the index name. -- Updated example distributor node config to match values currently used in production. ## 1.4.1 diff --git a/distributor-node/src/services/logging/LoggingService.ts b/distributor-node/src/services/logging/LoggingService.ts index ca4492c6a2..8ab155b8eb 100644 --- a/distributor-node/src/services/logging/LoggingService.ts +++ b/distributor-node/src/services/logging/LoggingService.ts @@ -86,8 +86,9 @@ export class LoggingService { let esTransport: ElasticsearchTransport | undefined if (config.logs?.elastic) { const indexPrefix = config.logs.elastic.indexPrefix || 'logs-argus' + const index = `${indexPrefix}-${config.id}`.toLowerCase() esTransport = new ElasticsearchTransport({ - index: `${indexPrefix}-${config.id}`, + index, dataStream: true, level: config.logs.elastic.level, format: winston.format.combine(pauseFormat({ id: 'es' }), escFormat()), diff --git a/storage-node/CHANGELOG.md b/storage-node/CHANGELOG.md index 52adb9ddf8..0e2a9a7c03 100644 --- a/storage-node/CHANGELOG.md +++ b/storage-node/CHANGELOG.md @@ -1,9 +1,12 @@ +### 3.8.0 + +- Changed Elasticsearch transport to use data streams instead of regular indices. Removed `--elasticSearchIndex` option and replaced with `--elasticSearchIndexPrefix`. Node ID from config will be automatically appended to the index name. + ### 3.7.2 -- Bumped `winston-elasticsearch` package verion +- Bumped `winston-elasticsearch` package verion - **FIX**: Added error handler to caught exception in `ElasticsearchTransport` and gracefully log them - ### 3.7.1 - Disable open-api express response validation if NODE_ENV == 'production'. This should improve response times when serving assets. diff --git a/storage-node/README.md b/storage-node/README.md index b7eea18319..0a755ba176 100644 --- a/storage-node/README.md +++ b/storage-node/README.md @@ -6,17 +6,18 @@ Joystream storage subsystem. ![License](https://img.shields.io/github/license/Joystream/joystream) -* [Colossus v2](#colossus-v2) -* [Description](#description) -* [Installation](#installation) -* [Ubuntu Linux](#ubuntu-linux) -* [Install packages required for installation](#install-packages-required-for-installation) -* [Clone the code repository](#clone-the-code-repository) -* [Install volta](#install-volta) -* [Install project dependencies and build it](#install-project-dependencies-and-build-it) -* [Verify installation](#verify-installation) -* [Usage](#usage) -* [CLI Commands](#cli-commands) + +- [Colossus v2](#colossus-v2) +- [Description](#description) +- [Installation](#installation) +- [Ubuntu Linux](#ubuntu-linux) +- [Install packages required for installation](#install-packages-required-for-installation) +- [Clone the code repository](#clone-the-code-repository) +- [Install volta](#install-volta) +- [Install project dependencies and build it](#install-project-dependencies-and-build-it) +- [Verify installation](#verify-installation) +- [Usage](#usage) +- [CLI Commands](#cli-commands) # Description @@ -147,28 +148,29 @@ There is also an option to run Colossus as [Docker container](../colossus.Docker # CLI Commands -* [`storage-node dev:multihash`](#storage-node-devmultihash) -* [`storage-node dev:sync`](#storage-node-devsync) -* [`storage-node dev:verify-bag-id`](#storage-node-devverify-bag-id) -* [`storage-node help [COMMAND]`](#storage-node-help-command) -* [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) -* [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) -* [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) -* [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) -* [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) -* [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) -* [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) -* [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) -* [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) -* [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) -* [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) -* [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) -* [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) -* [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) -* [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) -* [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) -* [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) -* [`storage-node server`](#storage-node-server) + +- [`storage-node dev:multihash`](#storage-node-devmultihash) +- [`storage-node dev:sync`](#storage-node-devsync) +- [`storage-node dev:verify-bag-id`](#storage-node-devverify-bag-id) +- [`storage-node help [COMMAND]`](#storage-node-help-command) +- [`storage-node leader:cancel-invite`](#storage-node-leadercancel-invite) +- [`storage-node leader:create-bucket`](#storage-node-leadercreate-bucket) +- [`storage-node leader:delete-bucket`](#storage-node-leaderdelete-bucket) +- [`storage-node leader:invite-operator`](#storage-node-leaderinvite-operator) +- [`storage-node leader:remove-operator`](#storage-node-leaderremove-operator) +- [`storage-node leader:set-bucket-limits`](#storage-node-leaderset-bucket-limits) +- [`storage-node leader:set-global-uploading-status`](#storage-node-leaderset-global-uploading-status) +- [`storage-node leader:update-bag-limit`](#storage-node-leaderupdate-bag-limit) +- [`storage-node leader:update-bags`](#storage-node-leaderupdate-bags) +- [`storage-node leader:update-blacklist`](#storage-node-leaderupdate-blacklist) +- [`storage-node leader:update-bucket-status`](#storage-node-leaderupdate-bucket-status) +- [`storage-node leader:update-data-fee`](#storage-node-leaderupdate-data-fee) +- [`storage-node leader:update-data-object-bloat-bond`](#storage-node-leaderupdate-data-object-bloat-bond) +- [`storage-node leader:update-dynamic-bag-policy`](#storage-node-leaderupdate-dynamic-bag-policy) +- [`storage-node leader:update-voucher-limits`](#storage-node-leaderupdate-voucher-limits) +- [`storage-node operator:accept-invitation`](#storage-node-operatoraccept-invitation) +- [`storage-node operator:set-metadata`](#storage-node-operatorset-metadata) +- [`storage-node server`](#storage-node-server) ## `storage-node dev:multihash` @@ -845,7 +847,7 @@ OPTIONS -z, --logFileChangeFrequency=(yearly|monthly|daily|hourly|none) [default: daily] Log files update frequency. - --elasticSearchIndex=elasticSearchIndex Elasticsearch index name. + --elasticSearchIndexPrefix=elasticSearchIndexPrefix Elasticsearch index prefix. Node ID will be appended to the prefix. Default: logs-colossus. --elasticSearchPassword=elasticSearchPassword Elasticsearch password for basic authentication. @@ -856,4 +858,5 @@ OPTIONS ``` _See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/master/src/commands/server.ts)_ + diff --git a/storage-node/package.json b/storage-node/package.json index a8f063e230..3c5b3e23e8 100644 --- a/storage-node/package.json +++ b/storage-node/package.json @@ -1,7 +1,7 @@ { "name": "storage-node", "description": "Joystream storage subsystem.", - "version": "3.7.2", + "version": "3.8.0", "author": "Joystream contributors", "bin": { "storage-node": "./bin/run" @@ -160,7 +160,7 @@ "ensure": "yarn format && yarn lint --fix && yarn build", "checks": "tsc --noEmit --pretty && prettier ./src --check && yarn lint", "start": "./bin/run server", - "start:with-instrumentation": "export OTEL_APPLICATION=storage-node; node --require @joystream/opentelemetry ./bin/run server" + "start:with-instrumentation": "export OTEL_APPLICATION=storage-node; export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8200; export OTEL_RESOURCE_ATTRIBUTES=service.name=colossus,deployment.environment=production; node --require @joystream/opentelemetry ./bin/run server" }, "types": "lib/index.d.ts" } diff --git a/storage-node/src/commands/server.ts b/storage-node/src/commands/server.ts index 7a0aa2b628..531d2b4b08 100644 --- a/storage-node/src/commands/server.ts +++ b/storage-node/src/commands/server.ts @@ -88,10 +88,10 @@ export default class Server extends ApiCommandBase { Log level could be set using the ELASTIC_LOG_LEVEL enviroment variable. Supported values: warn, error, debug, info. Default:debug`, }), - elasticSearchIndex: flags.string({ + elasticSearchIndexPrefix: flags.string({ required: false, - env: 'ELASTIC_INDEX', - description: 'Elasticsearch index name.', + env: 'ELASTIC_INDEX_PREFIX', + description: 'Elasticsearch index prefix. Node ID will be appended to the prefix. Default: logs-colossus', }), elasticSearchUser: flags.string({ dependsOn: ['elasticSearchEndpoint', 'elasticSearchPassword'], @@ -139,7 +139,7 @@ Supported values: warn, error, debug, info. Default:debug`, initNewLogger({ elasticSearchlogSource: logSource, elasticSearchEndpoint: flags.elasticSearchEndpoint, - elasticSearchIndex: flags.elasticSearchIndex, + elasticSearchIndexPrefix: flags.elasticSearchIndexPrefix, elasticSearchUser: flags.elasticSearchUser, elasticSearchPassword: flags.elasticSearchPassword, filePath: flags.logFilePath, diff --git a/storage-node/src/services/logger.ts b/storage-node/src/services/logger.ts index e4eadc7955..cdd4df5165 100644 --- a/storage-node/src/services/logger.ts +++ b/storage-node/src/services/logger.ts @@ -148,7 +148,7 @@ function createCustomLogger(customOptions: LogConfig): winston.Logger { createElasticTransport( customOptions.elasticSearchlogSource, customOptions.elasticSearchEndpoint, - customOptions.elasticSearchIndex, + customOptions.elasticSearchIndexPrefix, customOptions.elasticSearchUser, customOptions.elasticSearchPassword ) @@ -210,6 +210,9 @@ function createElasticTransport( elasticLogLevel = 'debug' // default } + const indexPrefix = elasticSearchIndex || 'logs-colossus' + const index = `${indexPrefix}-${logSource}`.toLowerCase() + const esTransport = new ElasticsearchTransport({ level: elasticLogLevel, clientOpts: { @@ -224,10 +227,36 @@ function createElasticTransport( } : {}), }, - index: elasticSearchIndex || 'storage-node', + index, + dataStream: true, format: ecsformat(), source: logSource, retryLimit: 10, + // apply custom transform so that tracing data (if present) is placed in the top level of the log + // based on https://github.com/vanthome/winston-elasticsearch/blob/d948fa1b705269a4713480593ea657de34c0a942/transformer.js + transformer: (logData) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const transformed: any = {} + transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString() + transformed.message = logData.message + transformed.severity = logData.level + transformed.fields = logData.meta + + if (logData.meta.trace_id || logData.meta.trace_flags) { + transformed.trace = { + id: logData.meta.trace_id, + flags: logData.meta.trace_flags, + } + } + if (logData.meta.span_id) { + transformed.span = { id: logData.meta.span_id } + } + if (logData.meta.transaction_id) { + transformed.transaction = { id: logData.meta.transaction_id } + } + + return transformed + }, }) // Handle ES logger error. @@ -310,8 +339,8 @@ export type LogConfig = { /** Elastic search engine endpoint */ elasticSearchEndpoint?: string - /** Elastic search index name */ - elasticSearchIndex?: string + /** Elastic search index prefix */ + elasticSearchIndexPrefix?: string /** Elastic search user */ elasticSearchUser?: string From 86a96be0362bf32fd4d847878775aae41311ee69 Mon Sep 17 00:00:00 2001 From: Klaudiusz Dembler Date: Thu, 16 Nov 2023 14:19:40 +0100 Subject: [PATCH 3/4] rename argument in colossus createElasticTransport --- storage-node/src/services/logger.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage-node/src/services/logger.ts b/storage-node/src/services/logger.ts index cdd4df5165..d66c582ae7 100644 --- a/storage-node/src/services/logger.ts +++ b/storage-node/src/services/logger.ts @@ -197,7 +197,7 @@ export function initNewLogger(options: LogConfig): void { function createElasticTransport( logSource: string, elasticSearchEndpoint: string, - elasticSearchIndex?: string, + elasticSearchIndexPrefix?: string, elasticSearchUser?: string, elasticSearchPassword?: string ): winston.transport { @@ -210,7 +210,7 @@ function createElasticTransport( elasticLogLevel = 'debug' // default } - const indexPrefix = elasticSearchIndex || 'logs-colossus' + const indexPrefix = elasticSearchIndexPrefix || 'logs-colossus' const index = `${indexPrefix}-${logSource}`.toLowerCase() const esTransport = new ElasticsearchTransport({ From 5499c01b8be534a16160fce6894d7c11e0289ff2 Mon Sep 17 00:00:00 2001 From: Klaudiusz Dembler Date: Thu, 16 Nov 2023 18:06:20 +0100 Subject: [PATCH 4/4] add sleep --- tests/network-tests/start-storage.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/network-tests/start-storage.sh b/tests/network-tests/start-storage.sh index 91b3f06acb..7a1ae378ff 100755 --- a/tests/network-tests/start-storage.sh +++ b/tests/network-tests/start-storage.sh @@ -12,6 +12,10 @@ export COLOSSUS_2_URL="http://${HOST_IP}:3335" export DISTRIBUTOR_2_URL="http://${HOST_IP}:3336" $THIS_DIR/run-test-scenario.sh initStorageAndDistribution + +# give QN time to catch up so nodes can get their initial state +sleep 30 + # Start colossus & argus docker-compose -f $THIS_DIR/../../docker-compose.yml up -d colossus-1 docker-compose -f $THIS_DIR/../../docker-compose.yml up -d distributor-1 @@ -29,4 +33,4 @@ docker logs colossus-2 --tail 300 echo "## distributor-1" docker logs distributor-1 --tail 300 echo "## distributor-2" -docker logs distributor-2 --tail 300 \ No newline at end of file +docker logs distributor-2 --tail 300