Add cardano-tracer and tracer-sidecar images

.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.hs]
+indent_size = 4
+max_line_length = 80
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space

.gitignore

@@ -2,6 +2,14 @@
 *~
 *.swp
 *.swo
+*.swl
+*.swm
+*.swn
+
 
 # compose
 compose/.testnet.yaml
+\#*
+
+# haskell
+dist-newstyle

compose/Makefile

@@ -44,7 +44,7 @@ help:
 	@echo "    make anti testnet=example_10.2.1 password='password1234'"
 	@echo
 
-build: TESTNET build-image build-config build-sidecar ## Build cardano-node, config and sidecar container image
+build: TESTNET build-image build-config build-sidecar build-tracer build-tracer-sidecar ## Build cardano-node, config, sidecar, and tracer images
 
 build-image: TESTNET ## Build cardano-node container image
 	ln -snf testnets/${testnet}/testnet.yaml .testnet.yaml ; cd testnets/${testnet} ; docker compose build
@@ -55,7 +55,13 @@ build-config: TESTNET ## Build config container image
 build-sidecar: TESTNET ## Build sidecar container image
 	docker build -f sidecar/Dockerfile -t ${registry}${testnet}_sidecar:latest sidecar/
 
-push: TESTNET push-image push-config push-sidecar ## Push cardano-node, config and sidecar container image
+build-tracer: TESTNET ## Build tracer container image
+	docker build -f tracer/Dockerfile -t ${registry}${testnet}_tracer:latest tracer/
+
+build-tracer-sidecar: TESTNET ## Build tracer container image
+	docker build -f tracer-sidecar/Dockerfile -t ${registry}${testnet}_tracer-sidecar:latest tracer-sidecar/
+
+push: TESTNET push-image push-config push-tracer push-sidecar push-tracer-sidecar ## Push cardano-node, config and sidecar container image
 
 push-image: TESTNET ## Push cardano-node container image
 	docker push ${registry}${testnet}:latest
@@ -66,6 +72,12 @@ push-config: TESTNET ## Push config container image
 push-sidecar: TESTNET ## Push sidecar container image
 	docker push ${registry}${testnet}_sidecar:latest
 
+push-tracer: TESTNET ## Push tracer container image
+	docker push ${registry}${testnet}_tracer:latest
+
+push-tracer-sidecar: TESTNET ## Push tracer container image
+	docker push ${registry}${testnet}_tracer-sidecar:latest
+
 up: TESTNET ## Start Run local Docker Compose
 	cd testnets/${testnet} ; docker compose up --detach
 
@@ -84,7 +96,7 @@ anti: TESTNET ## Run Antithesis job
 	--arg description   '${description}' \
 	--arg duration      '${duration}' \
 	--arg config_image  '${testnet}_config:latest' \
-	--arg images        '${testnet}:latest;${testnet}_sidecar:latest' \
+	--arg images        '${testnet}:latest;${testnet}_sidecar:latest;${testnet}_tracer:latest;${testnet}_tracer-sidecar:latest' \
 	--arg recipients    '${recipients}' \
 	'{params: {"antithesis.description": $$description,"custom.duration": $$duration,"antithesis.config_image": $$config_image,"antithesis.images": $$images,"antithesis.report.recipients": $$recipients}}')"
 

compose/README.md

@@ -75,7 +75,7 @@ Docker and `make` are the only requirement to build and run your own testnets.
 
 ### Local
 
-- Build the `cardano-node`, `config` and `sidecar` container images
+- Build the `cardano-node`, `cardano-tracer`, `config` and `sidecar` container images
 
   ```
   make build testnet=example_10.2.1
@@ -86,14 +86,15 @@ Docker and `make` are the only requirement to build and run your own testnets.
 
 ### Antithesis
 
-- Build the `cardano-node`, `config` and `sidecar` container images for the Antithesis container registry
+- Build the `cardano-node`, `cardano-tracer`, `config` and `sidecar` container images for the Antithesis container registry
 
   ```
   make build testnet=example_10.2.1 registry=us-central1-docker.pkg.dev/molten-verve-216720/cardano-repository/
   ```
 
 > [!IMPORTANT]
 > Always supply the registry argument when building for Antithesis.
+> It's not necessary to log into the registry for building, but it is necessary for running antithesis tests.
 
 ## Run
 
@@ -191,7 +192,7 @@ Docker and `make` are the only requirement to build and run your own testnets.
   cat credentials.json | docker login -u _json_key https://us-central1-docker.pkg.dev --password-stdin
   ```
 
-- Push the `cardano-node`, `config` and `sidecar` container images
+- Push the `cardano-node`, `cardano-tracer`, `config` and `sidecar` container images
 
   ```
   make push testnet=example_10.2.1 registry=us-central1-docker.pkg.dev/molten-verve-216720/cardano-repository/
@@ -230,6 +231,67 @@ Docker and `make` are the only requirement to build and run your own testnets.
 > [!TIP]
 > The commands above assume a testnet of 3 pools, increase the `{1..3}` if needed.
 
+### Logging & Metrics
+
+The `tracer` container which runs alongside the Cardano cluster aggregates logs and metrics for each of the running nodes. The details of the configuration are out of scope for this document, please checkout the [cardano-tracer](https://github.com/IntersectMBO/cardano-node/tree/master/cardano-tracer) and [new tracing system](https://developers.cardano.org/docs/get-started/cardano-node/new-tracing-system/new-tracing-system) documentation. The default example configuration aggregates some interesting logs and expose Prometheus metrics on port 4000.
+
+- Query prometheus metrics (each node has its own path for metrics):
+
+  ```bash
+  curl -s http://localhost:4000/p1example-3001 | grep block
+  ```
+
+  ```
+  # TYPE cardano_node_metrics_served_block_latest_int gauge
+  cardano_node_metrics_served_block_latest_int 4
+  # TYPE cardano_node_metrics_blockfetchclient_blocksize_int gauge
+  cardano_node_metrics_blockfetchclient_blocksize_int 862
+  # TYPE cardano_node_metrics_blocksForged_int gauge
+  cardano_node_metrics_blocksForged_int 2
+  cardano_node_metrics_blockfetchclient_blockdelay_real 4.574799e-3
+  # TYPE cardano_node_metrics_blockNum_int gauge
+  cardano_node_metrics_blockNum_int 4
+  # TYPE cardano_node_metrics_served_block_counter counter
+  cardano_node_metrics_served_block_counter 4
+  ```
+
+- List available logs (logs from each node are dumped into a file on a shared volume, the name of the file contains timestamp):
+
+  ```bash
+  docker exec -ti tracer ls -l  /opt/cardano-tracer
+  ```
+
+  ```
+  drwxr-xr-x 2 cardano cardano 4096 Apr 13 07:42 p1.example_3001
+  drwxr-xr-x 2 cardano cardano 4096 Apr 13 07:42 p2.example_3001
+  drwxr-xr-x 2 cardano cardano 4096 Apr 13 07:42 p3.example_3001
+  srwxr-xr-x 1 cardano cardano    0 Apr 13 07:42 tracer.socket
+  ```
+
+  ```bash
+  for i in {1..3}; do docker exec -ti tracer ls -l  /opt/cardano-tracer/p${i}.example_3001/ ; done
+  ```
+
+  ```
+  total 404
+  -rw-r--r-- 1 cardano cardano 411911 Apr 13 07:49 node-2025-04-13T07-42-11.json
+  total 376
+  -rw-r--r-- 1 cardano cardano 379328 Apr 13 07:49 node-2025-04-13T07-42-11.json
+  total 380
+  -rw-r--r-- 1 cardano cardano 384804 Apr 13 07:49 node-2025-04-13T07-42-10.json
+  ```
+
+- Follow logs from a specific node:
+
+  ```
+  $ docker exec -ti tracer tail -f  /opt/cardano-tracer/p2.example_3001/node-2025-04-13T07-42-11.json
+  {"at":"2025-04-13T07:47:57.001449232Z","ns":["Forge","Loop","NodeNotLeader"],"data":{"kind":"TraceNodeNotLeader","slot":417},"sev":"Info","thread":"41","host":"p2.example"}
+  {"at":"2025-04-13T07:47:58.001200722Z","ns":["Forge","Loop","StartLeadershipCheckPlus"],"data":{"chainDensity":3.7974683544303798610197731022708467207849025726318359375e-2,"delegMapSize":3,"kind":"TraceStartLeadershipCheck","slot":418,"utxoSize":3},"sev":"Info","thread":"41","host":"p2.example"}
+  {"at":"2025-04-13T07:47:58.001427133Z","ns":["Forge","StateInfo","StateInfo"],"data":{"credentials":"Cardano","endPeriod":0,"evolution":62,"kind":"KESInfo","startPeriod":0},"sev":"Info","thread":"41","host":"p2.example"}
+  {"at":"2025-04-13T07:47:58.001722819Z","ns":["Forge","Loop","NodeNotLeader"],"data":{"kind":"TraceNodeNotLeader","slot":418},"sev":"Info","thread":"41","host":"p2.example"}
+  ...
+  ```
+
 ## Appendix
 
 - [Antithesis Documentation](https://antithesis.com/docs/)

compose/cardano-node.sh

@@ -167,6 +167,7 @@ assemble_command() {
     cmd+=(run)
     cmd+=(--database-path ${DATABASE_PATH})
     cmd+=(--socket-path ${SOCKET_PATH})
+    cmd+=(--tracer-socket-path-connect /opt/cardano-tracer/tracer.socket)
 
     # TYPE
     if [ "${TYPE,,}" = "bp" ]; then

compose/testnets/anviking_sometimes_forks/README.md

@@ -0,0 +1,13 @@
+# Description
+
+Example 3 pool testnet with pre-compiled cardano-node version 10.2.1.
+
+## Cardano-Node
+
+- **Version**: 10.2.1
+- **Branch**: -
+- **Source/Compiled**: Compiled
+
+## Testnet
+
+- **Pools**: 3

compose/testnets/anviking_sometimes_forks/docker-compose.yaml

@@ -0,0 +1,93 @@
+---
+
+x-base: &base
+  image: ${registry}${testnet}:latest
+  restart: on-failure
+  build:
+    context: "../../"
+    dockerfile: "Dockerfile.compiled"
+    args:
+      CARDANO_NODE_VERSION: "10.2.1"
+
+x-env: &env
+  POOL_ID: "0" # Placeholder required for override
+
+services:
+  tracer:
+    image: ${registry}${testnet}_tracer:latest
+    restart: on-failure
+    hostname: tracer.example
+    container_name: tracer
+    volumes:
+      - tracer:/opt/cardano-tracer
+    ports:
+      - "4000:4000"
+    command:
+       - "--config"
+       - "tracer-config.yaml"
+    build:
+      context: "../../tracer"
+      dockerfile: "Dockerfile"
+
+  p1:
+    <<: *base
+    container_name: p1
+    hostname: p1.example
+    volumes:
+      - p1:/opt/cardano-node/data
+      - tracer:/opt/cardano-tracer
+    ports:
+      - "3001:3001"
+    environment:
+      <<: *env
+      POOL_ID: "1"
+
+  p2:
+    <<: *base
+    container_name: p2
+    hostname: p2.example
+    volumes:
+      - p2:/opt/cardano-node/data
+      - tracer:/opt/cardano-tracer
+    ports:
+      - "3002:3001"
+    environment:
+      <<: *env
+      POOL_ID: "2"
+      PEER_SHARING: "false"
+
+  p3:
+    <<: *base
+    container_name: p3
+    hostname: p3.example
+    volumes:
+      - p3:/opt/cardano-node/data
+      - tracer:/opt/cardano-tracer
+    ports:
+      - "3003:3001"
+    environment:
+      <<: *env
+      POOL_ID: "3"
+
+  s1:
+    image: ${registry}${testnet}_sidecar:latest
+    restart: on-failure
+    container_name: s1
+    hostname: s1.example
+    environment:
+      POOLS: "3"
+  tracer-sidecar:
+    image: ${registry}${testnet}_tracer-sidecar:latest
+    restart: on-failure
+    container_name: tracer-sidecar
+    hostname: tracer-sidecar.example
+    environment:
+      POOLS: "3"
+    volumes:
+      - tracer:/opt/cardano-tracer
+
+volumes:
+  tracer:
+  p1:
+  p2:
+  p3: