Telemetry final changes

.github/workflows/joystream-apps-docker.yml

@@ -2,6 +2,7 @@ name: Publish Colossus/Argus/QueryNode Docker images
 
 on:
   workflow_dispatch:
+  pull_request:
 
 jobs:
   build_and_publish:
@@ -26,6 +27,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Login to DockerHub
+        if: github.event_name == 'workflow_dispatch'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -58,67 +60,43 @@ jobs:
           push: false
           load: true
           tags: joystream/distributor-node:latest
-
-      - name: Test with latest images
-        run: |
-          yarn build:packages
-          cp docker-compose-no-bind-volumes.yml docker-compose.yml
-          ./tests/network-tests/run-tests.sh content-directory
-
-      - name: Push storage-node
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: colossus.Dockerfile
-          push: true
-          tags: joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
-
-      - name: Push distributor-node
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: distributor-node.Dockerfile
-          push: true
-          tags: joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
-
-      - name: Push query-node
+      - name: Build query-node
         uses: docker/build-push-action@v3
         with:
           # Do not use local dir context to ensure we can build from a commit directly
           # context: .
           file: query-node.Dockerfile
-          push: true
-          tags: joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          push: false
+          load: true
+          tags: joystream/query-node:latest
 
-      # Push latest tag if on master branch
-      - name: Push joystream/storage-node:latest
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: colossus.Dockerfile
-          push: true
-          tags: joystream/storage-node:latest
+      - name: Test with latest images
+        run: |
+          yarn build:packages
+          cp docker-compose-no-bind-volumes.yml docker-compose.yml
+          ./tests/network-tests/run-test-node-docker.sh
+          sleep 15
+          docker compose up -d db indexer hydra-indexer-gateway processor graphql-server
+          sleep 30
+          ./tests/network-tests/start-storage.sh
+          export REUSE_KEYS=true
+          export SKIP_STORAGE_AND_DISTRIBUTION=true
+          IGNORE_HIRED_LEADS=true ./tests/network-tests/run-test-scenario.sh content-directory
 
-      - name: Push joystream/distributor-node:latest
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: distributor-node.Dockerfile
-          push: true
-          tags: joystream/distributor-node:latest
+      - name: Push new versions
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          docker image tag joystream/storage-node:latest joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker image tag joystream/distributor:latest joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker image tag joystream/query-node:latest joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+          docker push joystream/storage-node:${{ steps.extract_versions.outputs.colossus_version }}
+          docker push joystream/distributor-node:${{ steps.extract_versions.outputs.argus_version }}
+          docker push joystream/query-node:${{ steps.extract_versions.outputs.qn_version }}
+
+      - name: Push latest tag
+        if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch'
+        run: |
+          docker push joystream/storage-node:latest
+          docker push joystream/distributor:latest
+          docker push joystream/query-node:latest
 
-      - name: Push query-node image
-        if: github.ref == 'refs/heads/master'
-        uses: docker/build-push-action@v3
-        with:
-          # Do not use local dir context to ensure we can build from a commit directly
-          # context: .
-          file: query-node.Dockerfile
-          push: true
-          tags: joystream/query-node:latest

distributor-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@joystream/distributor-cli",
   "description": "Joystream distributor node CLI",
-  "version": "1.2.0",
+  "version": "1.2.1",
   "author": "Joystream contributors",
   "bin": {
     "joystream-distributor": "./bin/run"

docker-compose-no-bind-volumes.yml

@@ -14,6 +14,9 @@ services:
 
   colossus-1:
     image: joystream/storage-node
+    build:
+      context: .
+      dockerfile: colossus.Dockerfile
     container_name: colossus-1
     restart: on-failure
     volumes:
@@ -31,6 +34,7 @@ services:
       - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI}
       - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200
       - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production
+    entrypoint: ['yarn']
     command: [
       'start:with-instrumentation', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data',
       '--sync', '--syncInterval=1',
@@ -41,6 +45,9 @@ services:
 
   distributor-1:
     image: joystream/distributor-node
+    build:
+      context: .
+      dockerfile: distributor-node.Dockerfile
     container_name: distributor-1
     restart: on-failure
     volumes:
@@ -74,6 +81,7 @@ services:
       # JOYSTREAM_DISTRIBUTOR__LOGS__ELASTIC: "{\"level\":\"debug\",\"endpoint\":\"http://elasticsearch:9200\"}"
     #   JOYSTREAM_DISTRIBUTOR__LIMITS__STORAGE: 50G
     #   JOYSTREAM_DISTRIBUTOR__BUCKETS: "[\"1:0\",\"1:1\"]"
+    entrypoint: ['yarn']
     command: ['start:with-instrumentation']
 
   colossus-2:
@@ -177,7 +185,7 @@ services:
     image: joystream/query-node
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: query-node.Dockerfile
     container_name: graphql-server
     restart: unless-stopped
     env_file:
@@ -196,15 +204,12 @@ services:
       - "127.0.0.1:${PROCESSOR_STATE_APP_PORT}:${PROCESSOR_STATE_APP_PORT}"
     depends_on:
       - db
-    volumes:
-      - type: bind
-        source: .
-        target: /joystream
     working_dir: /joystream
-    command: ['yarn', 'workspace', 'query-node-root', 'query-node:start:prod:with-instrumentation']
+    entrypoint: ['yarn']
+    command: ['workspace', 'query-node-root', 'query-node:start:prod:with-instrumentation']
 
   processor:
-    image: node:14
+    image: joystream/query-node
     container_name: processor
     restart: unless-stopped
     env_file:
@@ -217,14 +222,21 @@ services:
       - BATCH_SIZE=100
       - QUEUE_FACTOR=1
       - QUEUE_MAX_CAP_FACTOR=4
+      - WARTHOG_DB_HOST=db
+      - WARTHOG_DB_DATABASE=${DB_NAME}
+      - WARTHOG_DB_USERNAME=${DB_USER}
+      - WARTHOG_DB_PASSWORD=${DB_PASS}
+      - WARTHOG_APP_PORT=4002
     depends_on:
       - db
-    volumes:
-      - type: bind
-        source: .
-        target: /joystream
     working_dir: /joystream
-    command: ['yarn', 'workspace', 'query-node-root', 'processor:start']
+    entrypoint: sh
+    command: >
+      -c "
+        yarn workspace query-node-root db:prepare ;
+        yarn workspace query-node-root db:migrate ;
+        yarn workspace query-node-root processor:start
+      "
 
   indexer:
     image: joystream/hydra-indexer:v5.0.0-alpha.1

query-node.Dockerfile

@@ -10,6 +10,7 @@ RUN yarn --frozen-lockfile
 RUN yarn workspace @joystream/types build
 RUN yarn workspace @joystream/metadata-protobuf build
 RUN yarn workspace @joystream/js build
+RUN yarn workspace @joystream/opentelemetry build
 RUN yarn workspace query-node-root build
 
 ENTRYPOINT [ "yarn" ]