From 6c24821f57f6896bec4b2c42e782eb1425477bf6 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Thu, 25 Sep 2025 14:16:41 -0400 Subject: [PATCH 1/4] Update prometheus params passed to run monitored tmpnet cmd --- .github/actions/c-chain-reexecution-benchmark/action.yml | 4 ++++ .../workflows/c-chain-reexecution-benchmark-container.yml | 7 ++++--- .../workflows/c-chain-reexecution-benchmark-gh-native.yml | 7 ++++--- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index bbe2bc38225d..b5b0e247139a 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -30,6 +30,10 @@ inputs: description: 'The duration of the AWS role to assume for S3 access.' required: true default: '43200' # 12 hours + prometheus-url: + description: 'The URL of the prometheus instance.' + required: true + default: '' prometheus-push-url: description: 'The push URL of the prometheus instance.' required: true diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index d95041a56eb9..2dffddbd606d 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -105,9 +105,10 @@ jobs: end-block: ${{ matrix.end-block }} block-dir-src: ${{ matrix.block-dir-src }} current-state-dir-src: ${{ matrix.current-state-dir-src }} - prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} - prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} - prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + prometheus_url: ${{ secrets.PROMETHEUS_URL || '' }} + prometheus_push_url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} + prometheus_username: ${{ secrets.PROMETHEUS_USERNAME || '' }} + prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index e5dc6d20679b..c3172f8f190c 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -92,9 +92,10 @@ jobs: end-block: ${{ matrix.end-block }} block-dir-src: ${{ matrix.block-dir-src }} current-state-dir-src: ${{ matrix.current-state-dir-src }} - prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} - prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} - prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + prometheus_url: ${{ secrets.PROMETHEUS_URL || '' }} + prometheus_push_url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} + prometheus_username: ${{ secrets.PROMETHEUS_USERNAME || '' }} + prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' From 2214b709b5e5be180121521a80232099c3511229 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Thu, 25 Sep 2025 14:30:55 -0400 Subject: [PATCH 2/4] fix up params --- .github/actions/c-chain-reexecution-benchmark/action.yml | 1 + .../workflows/c-chain-reexecution-benchmark-container.yml | 8 ++++---- .../workflows/c-chain-reexecution-benchmark-gh-native.yml | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/actions/c-chain-reexecution-benchmark/action.yml b/.github/actions/c-chain-reexecution-benchmark/action.yml index b5b0e247139a..001ee106e7bb 100644 --- a/.github/actions/c-chain-reexecution-benchmark/action.yml +++ b/.github/actions/c-chain-reexecution-benchmark/action.yml @@ -97,6 +97,7 @@ runs: BENCHMARK_OUTPUT_FILE=${{ env.BENCHMARK_OUTPUT_FILE }} \ RUNNER_NAME=${{ inputs.runner_name }} \ METRICS_ENABLED=true + prometheus_url: ${{ inputs.prometheus-url }} prometheus_push_url: ${{ inputs.prometheus-push-url }} prometheus_username: ${{ inputs.prometheus-username }} prometheus_password: ${{ inputs.prometheus-password }} diff --git a/.github/workflows/c-chain-reexecution-benchmark-container.yml b/.github/workflows/c-chain-reexecution-benchmark-container.yml index 2dffddbd606d..9dbf74c4ff59 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-container.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-container.yml @@ -105,10 +105,10 @@ jobs: end-block: ${{ matrix.end-block }} block-dir-src: ${{ matrix.block-dir-src }} current-state-dir-src: ${{ matrix.current-state-dir-src }} - prometheus_url: ${{ secrets.PROMETHEUS_URL || '' }} - prometheus_push_url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} - prometheus_username: ${{ secrets.PROMETHEUS_USERNAME || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} + prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} + prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' diff --git a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml index c3172f8f190c..607ae8fc61fb 100644 --- a/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml +++ b/.github/workflows/c-chain-reexecution-benchmark-gh-native.yml @@ -92,10 +92,10 @@ jobs: end-block: ${{ matrix.end-block }} block-dir-src: ${{ matrix.block-dir-src }} current-state-dir-src: ${{ matrix.current-state-dir-src }} - prometheus_url: ${{ secrets.PROMETHEUS_URL || '' }} - prometheus_push_url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} - prometheus_username: ${{ secrets.PROMETHEUS_USERNAME || '' }} - prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} + prometheus-url: ${{ secrets.PROMETHEUS_URL || '' }} + prometheus-push-url: ${{ secrets.PROMETHEUS_PUSH_URL || '' }} + prometheus-username: ${{ secrets.PROMETHEUS_USERNAME || '' }} + prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} push-github-action-benchmark: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.repository == 'ava-labs/avalanchego' && github.ref_name == 'master') }} aws-role: ${{ github.event.inputs.push-post-state != '' && secrets.AWS_S3_RW_ROLE || secrets.AWS_S3_READ_ONLY_ROLE }} aws-region: 'us-east-2' From c1a5172141bf46ae2a3f28d1684767ffda0a92c4 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Fri, 26 Sep 2025 11:57:56 -0400 Subject: [PATCH 3/4] Add check metrics exist and network UUID to re-execution tests --- tests/reexecute/c/vm_reexecute_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index 63b32eb2b80d..4ba38d7af376 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/ava-labs/coreth/plugin/factory" + "github.com/google/uuid" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -62,10 +63,12 @@ var ( executionTimeout time.Duration labelsArg string - labels = map[string]string{ + networkUUID string = uuid.NewString() + labels = map[string]string{ "job": "c-chain-reexecution", "is_ephemeral_node": "false", "chain": "C", + "network_uuid": networkUUID, } configKey = "config" @@ -547,7 +550,8 @@ func collectRegistry(tb testing.TB, name string, timeout time.Duration, gatherer ctx, cancel := context.WithTimeout(context.Background(), timeout) tb.Cleanup(cancel) - r.NoError(tmpnet.StartPrometheus(ctx, tests.NewDefaultLogger("prometheus"))) + logger := tests.NewDefaultLogger("prometheus") + r.NoError(tmpnet.StartPrometheus(ctx, logger)) server, err := tests.NewPrometheusServer(gatherer) r.NoError(err) @@ -567,6 +571,8 @@ func collectRegistry(tb testing.TB, name string, timeout time.Duration, gatherer return nil }(), )) + + r.NoError(tmpnet.CheckMetricsExist(ctx, logger, networkUUID)) }) sdConfigFilePath, err = tmpnet.WritePrometheusSDConfig(name, tmpnet.SDConfig{ From 2798fb50895e0c2fc11e4de350685acddf01ece2 Mon Sep 17 00:00:00 2001 From: Aaron Buchwald Date: Tue, 30 Sep 2025 13:30:04 -0400 Subject: [PATCH 4/4] fix collectRegistry ctx --- tests/reexecute/c/vm_reexecute_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/reexecute/c/vm_reexecute_test.go b/tests/reexecute/c/vm_reexecute_test.go index 4ba38d7af376..9738c1bc38aa 100644 --- a/tests/reexecute/c/vm_reexecute_test.go +++ b/tests/reexecute/c/vm_reexecute_test.go @@ -177,7 +177,7 @@ func benchmarkReexecuteRange( r.NoError(prefixGatherer.Register("avalanche_snowman", consensusRegistry)) if metricsEnabled { - collectRegistry(b, "c-chain-reexecution", time.Minute, prefixGatherer, labels) + collectRegistry(b, "c-chain-reexecution", prefixGatherer, labels) } log := tests.NewDefaultLogger("c-chain-reexecution") @@ -544,14 +544,14 @@ func newConsensusMetrics(registry prometheus.Registerer) (*consensusMetrics, err // collectRegistry starts prometheus and collects metrics from the provided gatherer. // Attaches the provided labels + GitHub labels if available to the collected metrics. -func collectRegistry(tb testing.TB, name string, timeout time.Duration, gatherer prometheus.Gatherer, labels map[string]string) { +func collectRegistry(tb testing.TB, name string, gatherer prometheus.Gatherer, labels map[string]string) { r := require.New(tb) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - tb.Cleanup(cancel) + startPromCtx, cancel := context.WithTimeout(context.Background(), tests.DefaultTimeout) + defer cancel() logger := tests.NewDefaultLogger("prometheus") - r.NoError(tmpnet.StartPrometheus(ctx, logger)) + r.NoError(tmpnet.StartPrometheus(startPromCtx, logger)) server, err := tests.NewPrometheusServer(gatherer) r.NoError(err) @@ -572,7 +572,9 @@ func collectRegistry(tb testing.TB, name string, timeout time.Duration, gatherer }(), )) - r.NoError(tmpnet.CheckMetricsExist(ctx, logger, networkUUID)) + checkMetricsCtx, cancel := context.WithTimeout(context.Background(), tests.DefaultTimeout) + defer cancel() + r.NoError(tmpnet.CheckMetricsExist(checkMetricsCtx, logger, networkUUID)) }) sdConfigFilePath, err = tmpnet.WritePrometheusSDConfig(name, tmpnet.SDConfig{