solana-validator-ha

failover:
  active:
    command: "set-identity-with-rollback.sh"
    args: [
      "--active-identity-file", "{{ .ActiveIdentityKeypairFile }}",
      "--passive-identity-file", "{{ .PassiveIdentityKeypairFile }}",
    ]

failover:
  passive:
    # ⚠️ This must make absolutely sure the validator goes passive.
    # ⚠️ If set-identity fails, restart/stop the service, pull the plug,
    # ⚠️ or call your mum crying for help. Do WHATEVER is necessary.
    command: "seppuku.sh"
    args: [
      "--passive-identity-file", "{{ .PassiveIdentityKeypairFile }}",
    ]

log:

  # required: false | default: info
  # Minimum log level. One of: debug, info, warn, error, fatal
  level: info

  # required: false | default: text
  # Log format. One of: text, logfmt, json
  format: text

validator:

  # required: true
  # Vanity name for this validator — used in logging and metrics
  name: "primary-validator"

  # required: false | default: http://localhost:8899
  # Local RPC URL for querying health and identity status
  rpc_url: "http://localhost:8899"

  # required: false | default: see internal/config/validator.go
  # List of URLs used to determine this node's public IPv4 address.
  # Each URL should return the IP as a plain string on the first line of the response.
  public_ip_service_urls: []

  identities:

    # required: true (or set active_pubkey)
    # Path to the active keypair file — shared across all HA peers.
    # Takes precedence over active_pubkey if both are set.
    active: "/path/to/active-identity.json"

    # required: true (or set active)
    # Base58-encoded active pubkey. Used when the keypair file is not available on this node.
    active_pubkey: 111111ActivePubkey1111111111111111111111111

    # required: true (or set passive_pubkey)
    # Path to the passive keypair file — unique per peer.
    # Takes precedence over passive_pubkey if both are set.
    passive: "/path/to/passive-identity.json"

    # required: true (or set passive)
    # Base58-encoded passive pubkey. Used when the keypair file is not available on this node.
    passive_pubkey: 111111PassivePubkey1111111111111111111111111

prometheus:

  # required: false | default: 9090
  # Port to serve Prometheus metrics on /metrics
  port: 9090

  # required: false | default: 9091
  # Port to serve the health check endpoint on /health
  health_check_port: 9091

  # required: false
  # Static key:value labels attached to all exposed metrics
  static_labels:
    brand: ha-validators
    cluster: mainnet-beta
    region: ha-region-1

cluster:

  # required: true
  # Solana cluster this validator is running on. One of: mainnet-beta, devnet, testnet
  name: "mainnet-beta"

  # required: false | default: cluster default RPC URL for cluster.name
  # RPC URLs used to query gossip state. Supplying multiple URLs provides resilience
  # against individual RPC drop-outs. URLs that return 403/429/503 are automatically
  # deprioritised for rpc_url_cooldown_duration before being retried.
  # The local validator RPC URL may be included here (logs a warning) and acts as a
  # rate-limit-immune fallback. For sub-second peer detection add it alongside at least
  # one remote URL and configure HA peers as mutual --entrypoint flags in agave.
  # See "Using Local RPC as a Failsafe Fallback" below for details.
  rpc_urls: []

  # required: false | default: 5s
  # Per-call timeout for cluster RPC requests. A hanging call is abandoned after this
  # duration and the next URL in the list is tried. Lower values detect slow endpoints
  # faster at the cost of spurious failures on genuinely loaded RPCs.
  # Public Solana RPC endpoints can take 1–2s under normal load — the 5s default leaves
  # comfortable headroom. Only lower this if you are using fast private/local RPCs.
  rpc_timeout_duration: 5s

  # required: false | default: 60s
  # How long a URL that returns 403/429/503 is deprioritised before being retried.
  # The public Solana RPC throttles on a per-minute basis, so 60s aligns with that
  # recovery window. Lower if your RPC provider recovers faster.
  rpc_url_cooldown_duration: 60s

failover:

  # required: false | default: false
  # When true, log failover commands instead of running them — useful for testing config.
  dry_run: false

  # required: false | default: 5s
  # How often to refresh gossip state and evaluate failover decisions.
  poll_interval_duration: 5s

  # required: false | default: 3  (~15s at the default 5s poll interval)
  # How many consecutive gossip samples without an active, non-delinquent voting peer
  # before the cluster is considered leaderless and a failover is triggered.
  leaderless_samples_threshold: 3

  # required: false | default: poll_interval_duration (no behaviour change)
  # How often to poll gossip for the final confirmatory sample once leaderless_samples_threshold-1
  # consecutive slow-poll misses have already been observed.
  #
  # The slow polls (poll_interval_duration) build confidence that the peer is genuinely down —
  # gossip can transiently drop a node for a single cycle and recover on the next. Fast-polling
  # only kicks in after threshold-1 independent slow-poll confirmations, so the risk of reacting
  # to a transient blip is very low by that point.
  #
  # Example with poll_interval_duration: 5s, leaderless_samples_threshold: 3,
  # leaderless_confirmation_poll_duration: 1s:
  #   T+5s   sample 1 (slow) → leaderless_count=1
  #   T+10s  sample 2 (slow) → leaderless_count=2  ← two independent slow confirmations
  #   T+11s  sample 3 (fast) → leaderless_count=3  → failover triggered
  #   Total: ~11s vs ~15s without this setting
  #
  # Constraints:
  #   - Must not exceed poll_interval_duration (ceiling — a slower confirmation poll is nonsensical)
  #   - Values below 1s are clamped to 1s on startup with a warning (floor — sub-second polling
  #     sits inside gossip propagation jitter and increases false-positive failover risk)
  leaderless_confirmation_poll_duration: 1s

  # required: false | default: false
  # When true, skips the leaderless sample threshold if the active peer is declared delinquent
  # by the network (and not due to a sub-rent-exempt vote account balance), triggering failover
  # immediately on the current gossip refresh.
  # ⚠️ Risk: a validator on a minority fork appears delinquent but may recover. Enabling this
  # removes the leaderless threshold recovery window and can cause unnecessary failovers.
  # See "Delinquency Fast-Path" below for the full trade-off analysis before enabling.
  delinquency_bypass: false

  # Overrides the number of slots a peer must be behind the tip to be considered delinquent.
  # ⚠️ Set with caution — too low a value causes false positives on transient hiccups.
  # The Agave default is 128 slots (~51s), defined as DELINQUENT_VALIDATOR_SLOT_DISTANCE:
  #   https://github.com/anza-xyz/agave/blob/master/rpc-client-types/src/request.rs
  # Since Agave v2.0, --health-check-slot-distance also defaults to 128 via the same constant:
  #   https://github.com/anza-xyz/agave/blob/master/validator/src/commands/run/args/json_rpc_config.rs
  # Both thresholds agree — there is no gap between delinquency detection and health status.
  # If you set value below 128, add --health-check-slot-distance <value> to your validator
  # startup flags to keep the thresholds aligned — a startup warning will remind you if not.
  # Values <= 1 are clamped to 2 on startup.
  delinquent_slot_distance_override:

    # required: false | default: false
    enabled: false

    # required: false | default: 128
    # Slots behind the tip at which a peer is considered delinquent (when enabled: true).
    value: 128

  # Guards against startup health flaps: a validator that briefly reports healthy during
  # startup before falling behind and going unhealthy again.
  self_healthy:

    # required: false | default: 30s
    # How long the local validator RPC must continuously report healthy before this
    # node is eligible to become active in a failover.
    minimum_duration: 30s

    # required: false | default: 2s
    # How often to sample local RPC health. Runs independently of poll_interval_duration
    # so the healthy streak timer is not skewed by gossip refresh latency.
    poll_interval_duration: 2s

    # required: false | default: 1
    # How many consecutive unhealthy samples are tolerated before the healthy streak is reset.
    # Default 1 means a single transient RPC timeout or blip does not destroy an established
    # streak — the streak is only reset if unhealthiness persists across back-to-back samples.
    # Set to 0 to disable grace (any failure resets immediately — original behaviour).
    unhealthy_grace_count: 1

  # Incident recording: writes JSON for network anomalies and failover decisions.
  recording:

    # required: false | default: false
    # When true, a JSON recording is checkpointed from the first anomaly through its outcome.
    enabled: false

    # required: false | default: directory of the loaded config file
    # Directory where recording files are written. Must exist and be writable — validated at startup.
    # If omitted, files land next to the config file.
    output_dir: "/var/log/solana-validator-ha/recordings"

  # required: true | min: 1
  # Map of HA peers, excluding this node — it is added automatically at startup.
  # Keys are vanity names used in logging and metrics. IPs must be valid, unique IPv4 addresses.
  peers:
    backup-validator-1:
      ip: 192.168.1.11
    backup-validator-2:
      ip: 192.168.1.12

  # required: false
  # Explicit failover priority for this node. Lower value = higher priority (takes over first).
  # When set, every peer in failover.peers must also declare a priority, and all values must be
  # unique and non-negative. If omitted (default), peers are ranked by ascending IP address.
  # All nodes in the HA cluster should declare the same priority ordering to avoid races.
  # See "Failover Priority" below for details.
  priority: 0

  # required: true
  # Commands and hooks to run when this node should become active.
  # command, args, and env values support Go template strings:
  #   {{ .ActiveIdentityKeypairFile }}  — absolute path to validator.identities.active
  #   {{ .PassiveIdentityKeypairFile }} — absolute path to validator.identities.passive
  #   {{ .ActiveIdentityPubkey }}       — active pubkey string
  #   {{ .PassiveIdentityPubkey }}      — passive pubkey string
  #   {{ .SelfName }}                   — value of validator.name
  active:

    # required: true
    command: set-identity-with-rollback.sh

    # required: false
    env:
      CUSTOM_ENV_VAR: "{{ .ActiveIdentityPubkey }}"

    # required: false
    args: [
      "active",
      "--active-identity-file",  "{{ .ActiveIdentityKeypairFile }}",
      "--passive-identity-file", "{{ .PassiveIdentityKeypairFile }}",
    ]

    # required: false
    # Hooks run in declaration order. A pre-hook with must_succeed: true aborts
    # subsequent hooks and skips the active command if it fails.
    hooks:
      pre:
        - name: notify-slack-promoting
          command: /home/solana/solana-validator-ha/hooks/pre-active/send-slack-alert.sh
          must_succeed: false
          env: {}
          args: [
            "--channel", "#save-my-bacon",
            "--message", "solana-validator-ha promoting {{ .SelfName }} to active ({{ .PassiveIdentityPubkey }} -> {{ .ActiveIdentityPubkey }})"
          ]

      post:
        - name: notify-slack-promoted
          command: /home/solana/solana-validator-ha/hooks/post-active/send-slack-alert.sh
          env: {}
          args: [
            "--channel", "#saved-my-bacon",
            "--message", "solana-validator-ha promoted {{ .SelfName }} to active with identity {{ .ActiveIdentityPubkey }}"
          ]

  # required: true
  # Commands and hooks to run when this node should become passive.
  # Supports the same template variables as active above.
  # This command must be idempotent — it may be called multiple times in succession.
  # post-passive hooks only run if the passive command succeeds.
  passive:

    # required: true
    command: seppuku.sh

    # required: false
    args: [
      "--passive-identity-file", "{{ .PassiveIdentityKeypairFile }}",
      "--stop-service-on-identity-set-failure",
      "--wait-for-and-force-identity-on-service-starting-up",
    ]

    # required: false
    hooks:
      pre:
        - name: notify-slack-demoting
          command: /home/solana/solana-validator-ha/hooks/pre-passive/send-slack-alert.sh
          must_succeed: false
          args: [
            "--channel", "#oh-shit-wake-people-up",
            "--message", "solana-validator-ha demoting {{ .SelfName }} to passive ({{ .ActiveIdentityPubkey }} -> {{ .PassiveIdentityPubkey }})"
          ]

      post:
        - name: notify-slack-demoted
          command: /home/solana/solana-validator-ha/hooks/post-passive/send-slack-alert.sh
          args: [
            "--channel", "#postmortem-shelf",
            "--message", "solana-validator-ha demoted {{ .SelfName }} to passive with identity {{ .PassiveIdentityPubkey }}"
          ]

update:

  # required: false | default: true
  # Set to false to disable all update checks (startup and periodic).
  check_enabled: true

  # required: false | default: 24h
  # How often to check for a new release when running in continuous mode.
  check_interval_duration: 24h

cluster:
  name: "mainnet-beta"
  rpc_urls:
    - "https://api.mainnet-beta.solana.com"  # remote — may rate-limit
    - "http://localhost:8899"                # local — immune to rate limits

failover:
  delinquency_bypass: true  # default: false

# On the node that should take over first (highest priority):
failover:
  priority: 0
  peers:
    backup-validator-1:
      ip: 192.168.1.11
      priority: 1
    backup-validator-2:
      ip: 192.168.1.12
      priority: 2

# On backup-validator-1:
failover:
  priority: 1
  peers:
    primary-validator:
      ip: 192.168.1.10
      priority: 0
    backup-validator-2:
      ip: 192.168.1.12
      priority: 2

failover:
  recording:
    enabled: true
    output_dir: "/var/log/solana-validator-ha/recordings"  # default: config file directory