Telemetry final changes

.github/workflows/joystream-apps-docker.yml

@@ -2,6 +2,7 @@ name: Publish Colossus/Argus/QueryNode Docker images
 
 on:
   workflow_dispatch:
+  pull_request:
 
 jobs:
   build_and_publish:
@@ -26,6 +27,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
+        if: github.event_name == 'workflow_dispatch'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,67 +60,43 @@ jobs:
           push: false
           load: true
           tags: joystream/distributor-node:latest
-
-      - name: Test with latest images
-        run: |
-          yarn build:packages
-          cp docker-compose-no-bind-volumes.yml docker-compose.yml
-          ./tests/network-tests/run-tests.sh content-directory
-
-      - name: Push storage-node
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: colossus.Dockerfile
-          push: true
-          tags: joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
-
-      - name: Push distributor-node
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: distributor-node.Dockerfile
-          push: true
-          tags: joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
-
-      - name: Push query-node
+      - name: Build query-node
         uses: docker/build-push-action@v3
         with:
           # Do not use local dir context to ensure we can build from a commit directly
           # context: .
           file: query-node.Dockerfile
-          push: true
-          tags: joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          push: false
+          load: true
+          tags: joystream/query-node:latest
 
-      # Push latest tag if on master branch
-      - name: Push joystream/storage-node:latest
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: colossus.Dockerfile
-          push: true
-          tags: joystream/storage-node:latest
+      - name: Test with latest images
+        run: |
+          yarn build:packages
+          cp docker-compose-no-bind-volumes.yml docker-compose.yml
+          ./tests/network-tests/run-test-node-docker.sh
+          sleep 15
+          docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
+          sleep 30
+          ./tests/network-tests/start-storage.sh
+          export REUSE_KEYS=true
+          export SKIP_STORAGE_AND_DISTRIBUTION=true
+          IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
 
-      - name: Push joystream/distributor-node:latest
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: distributor-node.Dockerfile
-          push: true
-          tags: joystream/distributor-node:latest
+      - name: Push new versions
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker image tag joystream/distributor:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+
+      - name: Push latest tag
+        if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
+        run: |
+          docker push joystream/storage-node:latest
+          docker push joystream/distributor:latest
+          docker push joystream/query-node:latest
 
-      - name: Push query-node image
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: query-node.Dockerfile
-          push: true
-          tags: joystream/query-node:latest

build-packages.sh

@@ -6,6 +6,7 @@ yarn --frozen-lockfile
 yarn workspace @joystream/types build
 yarn workspace @joystream/metadata-protobuf build
 yarn workspace @joystream/js build
+yarn workspace @joystream/opentelemetry build
 yarn workspace query-node-root build
 yarn workspace @joystream/cli build
 yarn workspace storage-node build

colossus.Dockerfile

@@ -10,6 +10,7 @@ COPY ./devops/eslint-config/package.json ./devops/eslint-config/package.json
 COPY ./devops/prettier-config/package.json ./devops/prettier-config/package.json
 COPY ./storage-node/package.json storage-node/package.json
 COPY ./storage-node/client/package.json storage-node/client/package.json
+COPY ./opentelemetry/package.json opentelemetry/package.json
 COPY ./package.json package.json
 
 RUN yarn --frozen-lockfile
@@ -19,10 +20,12 @@ COPY ./metadata-protobuf metadata-protobuf
 COPY ./devops/eslint-config ./devops/eslint-config
 COPY ./devops/prettier-config ./devops/prettier-config
 COPY ./storage-node storage-node
+COPY ./opentelemetry opentelemetry
 COPY ./tsconfig.json ./tsconfig.json
 
 RUN yarn workspace @joystream/types build
 RUN yarn workspace @joystream/metadata-protobuf build
+RUN yarn workspace @joystream/opentelemetry build
 RUN yarn workspace storage-node build
 RUN yarn cache clean
 
@@ -38,4 +41,4 @@ VOLUME ["/data", "/keystore", "/logs"]
 EXPOSE 3333
 
 ENTRYPOINT ["yarn", "storage-node"]
-CMD ["server"]
+CMD ["server"]

distributor-node.Dockerfile

@@ -12,6 +12,7 @@ COPY ./storage-node/package.json storage-node/package.json
 COPY ./storage-node/client/package.json storage-node/client/package.json
 COPY ./distributor-node/package.json distributor-node/package.json
 COPY ./distributor-node/client/package.json distributor-node/client/package.json
+COPY ./opentelemetry/package.json opentelemetry/package.json
 COPY ./package.json package.json
 
 RUN yarn --frozen-lockfile
@@ -22,6 +23,7 @@ COPY ./storage-node ./storage-node
 COPY ./devops/eslint-config ./devops/eslint-config
 COPY ./devops/prettier-config ./devops/prettier-config
 COPY ./distributor-node distributor-node
+COPY ./opentelemetry opentelemetry
 COPY ./tsconfig.json ./tsconfig.json
 
 # Build & cleanup
@@ -30,6 +32,7 @@ RUN \
   yarn workspace @joystream/types build &&\
   yarn workspace @joystream/metadata-protobuf build &&\
   yarn workspace @joystream/storage-node-client build &&\
+  yarn workspace @joystream/opentelemetry build &&\
   yarn workspace @joystream/distributor-cli build &&\
   yarn cache clean
 

distributor-node/CHANGELOG.md

@@ -1,5 +1,6 @@
-### 1.2.0
+### 1.2.1
 
+- Integrates OpenTelemetry API/SDK with Argus for exporting improved tracing logs & metrics to Elasticsearch. Adds `./start-elasticsearch-stack.sh` script to bootstrap elasticsearch services (Elasticsearch + Kibana + APM Server) with all the required configurations.
 - Add support for Elasticsearch authentication (`logs.elastic.auth` config section)
 - **FIX:** Objects belonging to group `0` of LRU-SP cache were never fully dropped from the state because of a bug in `StateCacheService.dropById`
 

distributor-node/client/api.ts

@@ -90,6 +90,12 @@ export interface StatusResponse {
      * @memberof StatusResponse
      */
     'version': string;
+    /**
+     * 
+     * @type {number}
+     * @memberof StatusResponse
+     */
+    'workerId'?: number;
     /**
      * 
      * @type {number}

distributor-node/docker-compose.yml

@@ -22,46 +22,9 @@ services:
       - 127.0.0.1:3334:3334
     working_dir: /joystream/distributor-node
     init: true
-    entrypoint: ['./bin/run']
-    command: ['start', './config/docker/distributor-dev.docker.yml']
-  # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/7.13/docker.html
-  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
-    container_name: elasticsearch
-    environment:
-      - discovery.type=single-node
-      - bootstrap.memory_lock=true
-      - 'ES_JAVA_OPTS=-Xms512m -Xmx512m'
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-    volumes:
-      - es-data:/usr/share/elasticsearch/data
-    ports:
-      - 127.0.0.1:9200:9200
-  # Ref: https://www.elastic.co/guide/en/kibana/7.13/docker.html
-  kibana:
-    image: docker.elastic.co/kibana/kibana:7.13.2
-    container_name: kibana
-    ports:
-      - 127.0.0.1:5601:5601
-    environment:
-      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
-  # Ref: https://www.elastic.co/guide/en/beats/filebeat/current/running-on-docker.html
-  filebeat:
-    user: root
-    image: joystream/distributor-filebeat
-    build:
-      context: ./config/docker
-      dockerfile: ./filebeat.Dockerfile
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - logs:/logs
+    command: ['yarn', 'start']
 
 volumes:
-  es-data:
-    driver: local
   logs:
     driver: local
   cache:

distributor-node/docs/schema/definition-properties-endpoints-properties-joystreamnodews.md

@@ -1,3 +1,7 @@
 ## joystreamNodeWs Type
 
 `string`
+
+## joystreamNodeWs Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")

distributor-node/docs/schema/definition-properties-endpoints-properties-querynode.md

@@ -1,3 +1,7 @@
 ## queryNode Type
 
 `string`
+
+## queryNode Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")

distributor-node/docs/schema/definition-properties-endpoints.md

@@ -27,6 +27,10 @@ Query node graphql server uri (for example: <http://localhost:8081/graphql>)
 
 `string`
 
+### queryNode Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")
+
 ## joystreamNodeWs
 
 Joystream node websocket api uri (for example: ws\://localhost:9944)
@@ -44,3 +48,7 @@ Joystream node websocket api uri (for example: ws\://localhost:9944)
 ### joystreamNodeWs Type
 
 `string`
+
+### joystreamNodeWs Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options-properties-endpoint.md

@@ -1,3 +1,7 @@
 ## endpoint Type
 
 `string`
+
+## endpoint Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")

distributor-node/docs/schema/definition-properties-logs-properties-elasticsearch-logging-options.md

@@ -61,6 +61,10 @@ Elastichsearch endpoint to push the logs to (for example: <http://localhost:9200
 
 `string`
 
+### endpoint Constraints
+
+**URI**: the string must be a URI, according to [RFC 3986](https://tools.ietf.org/html/rfc3986 "check the specification")
+
 ## index
 
 Elasticsearch index to push the logs to. If not provided, will fallback to "distributor-node"

distributor-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"
@@ -11,6 +11,7 @@
     "@apollo/client": "^3.2.5",
     "@elastic/ecs-winston-format": "^1.1.0",
     "@joystream/metadata-protobuf": "^2.8.0",
+    "@joystream/opentelemetry": "1.0.0",
     "@joystream/storage-node-client": "^3.0.0",
     "@joystream/types": "^2.0.0",
     "@oclif/command": "^1",
@@ -157,7 +158,9 @@
     "lint": "eslint ./src --ext .ts",
     "format": "prettier ./ --write",
     "checks": "tsc --noEmit --pretty && prettier ./ --check && yarn lint",
-    "cli": "./bin/run"
+    "cli": "./bin/run",
+    "start": "./bin/run start",
+    "start:with-instrumentation": "export OTEL_APPLICATION=distributor-node; node --require @joystream/opentelemetry ./bin/run start"
   },
   "types": "lib/index.d.ts"
 }