diff --git a/apps/hip-3-pusher/config/config.toml b/apps/hip-3-pusher/config/config.toml index dbbd5c3fae..9f8fc9ae36 100644 --- a/apps/hip-3-pusher/config/config.toml +++ b/apps/hip-3-pusher/config/config.toml @@ -1,4 +1,5 @@ stale_price_threshold_seconds = 5 +prometheus_port = 9090 [hyperliquid] market_name = "" @@ -15,7 +16,7 @@ aws_region_name = "ap-northeast-1" [lazer] lazer_urls = ["wss://pyth-lazer-0.dourolabs.app/v1/stream", "wss://pyth-lazer-1.dourolabs.app/v1/stream"] -api_key = "lazer_api_key" +lazer_api_key = "lazer_api_key" base_feed_id = 1 # BTC base_feed_exponent = -8 quote_feed_id = 8 # USDT diff --git a/apps/hip-3-pusher/pyproject.toml b/apps/hip-3-pusher/pyproject.toml index 1c557ead0f..e54ffd1074 100644 --- a/apps/hip-3-pusher/pyproject.toml +++ b/apps/hip-3-pusher/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hip-3-pusher" -version = "0.1.1" +version = "0.1.2" description = "Hyperliquid HIP-3 market oracle pusher" readme = "README.md" requires-python = ">=3.13" @@ -10,6 +10,9 @@ dependencies = [ "cryptography>=45.0.7", "hyperliquid-python-sdk>=0.19.0", "loguru>=0.7.3", + "opentelemetry-exporter-prometheus>=0.58b0", + "opentelemetry-sdk>=1.37.0", + "prometheus-client>=0.23.1", "toml>=0.10.2", "websockets>=15.0.1", ] diff --git a/apps/hip-3-pusher/src/main.py b/apps/hip-3-pusher/src/main.py index 5b4f703bde..c599e0806e 100644 --- a/apps/hip-3-pusher/src/main.py +++ b/apps/hip-3-pusher/src/main.py @@ -10,6 +10,7 @@ from hermes_listener import HermesListener from price_state import PriceState from publisher import Publisher +from metrics import Metrics def load_config(): @@ -38,7 +39,9 @@ async def main(): config = load_config() price_state = PriceState(config) - publisher = Publisher(config, price_state) + metrics = Metrics(config) + + publisher = Publisher(config, price_state, metrics) hyperliquid_listener = HyperliquidListener(config, price_state) lazer_listener = LazerListener(config, price_state) hermes_listener = HermesListener(config, price_state) diff --git a/apps/hip-3-pusher/src/metrics.py b/apps/hip-3-pusher/src/metrics.py new file mode 100644 index 0000000000..eb45c83858 --- /dev/null +++ b/apps/hip-3-pusher/src/metrics.py @@ -0,0 +1,37 @@ +from prometheus_client import start_http_server +from opentelemetry.exporter.prometheus import PrometheusMetricReader +from opentelemetry.metrics import get_meter_provider, set_meter_provider +from opentelemetry.sdk.metrics import MeterProvider + +METER_NAME = "hip3pusher" + + +class Metrics: + def __init__(self, config): + # Adapted from opentelemetry-exporter-prometheus example code. + # Start Prometheus client + start_http_server(port=config["prometheus_port"], addr="localhost") + # Exporter to export metrics to Prometheus + reader = PrometheusMetricReader() + # Meter is responsible for creating and recording metrics + set_meter_provider(MeterProvider(metric_readers=[reader])) + # TODO: sync version number and add? + self.meter = get_meter_provider().get_meter(METER_NAME) + + self._init_metrics() + + def _init_metrics(self): + self.no_oracle_price_counter = self.meter.create_counter( + name="hip_3_pusher_no_oracle_price_count", + description="Number of failed push attempts with no valid oracle price", + ) + self.successful_push_counter = self.meter.create_counter( + name="hip_3_pusher_successful_push_count", + description="Number of successful push attempts", + ) + self.failed_push_counter = self.meter.create_counter( + name="hip_3_pusher_failed_push_count", + description="Number of failed push attempts", + ) + + # TODO: labels/attributes diff --git a/apps/hip-3-pusher/src/publisher.py b/apps/hip-3-pusher/src/publisher.py index 53d2911fe5..23e5f90af1 100644 --- a/apps/hip-3-pusher/src/publisher.py +++ b/apps/hip-3-pusher/src/publisher.py @@ -8,6 +8,7 @@ from hyperliquid.utils.constants import TESTNET_API_URL, MAINNET_API_URL from kms_signer import KMSSigner +from metrics import Metrics from price_state import PriceState @@ -17,7 +18,7 @@ class Publisher: See https://hyperliquid.gitbook.io/hyperliquid-docs/for-developers/api/hip-3-deployer-actions """ - def __init__(self, config: dict, price_state: PriceState): + def __init__(self, config: dict, price_state: PriceState, metrics: Metrics): self.publish_interval = float(config["hyperliquid"]["publish_interval"]) self.kms_signer = None self.enable_kms = False @@ -43,40 +44,54 @@ def __init__(self, config: dict, price_state: PriceState): self.enable_publish = config["hyperliquid"].get("enable_publish", False) self.price_state = price_state + self.metrics = metrics async def run(self): while True: await asyncio.sleep(self.publish_interval) + try: + self.publish() + except Exception as e: + logger.exception("Publisher.publish() exception: {}", e) - oracle_pxs = {} - oracle_px = self.price_state.get_current_oracle_price() - if not oracle_px: - logger.error("No valid oracle price available!") - return - else: - logger.debug("Current oracle price: {}", oracle_px) - oracle_pxs[self.market_symbol] = oracle_px + def publish(self): + oracle_pxs = {} + oracle_px = self.price_state.get_current_oracle_price() + if not oracle_px: + logger.error("No valid oracle price available") + self.metrics.no_oracle_price_counter.add(1) + return + else: + logger.debug("Current oracle price: {}", oracle_px) + oracle_pxs[self.market_symbol] = oracle_px - mark_pxs = [] - #if self.price_state.hl_mark_price: - # mark_pxs.append({self.market_symbol: self.price_state.hl_mark_price}) + mark_pxs = [] + #if self.price_state.hl_mark_price: + # mark_pxs.append({self.market_symbol: self.price_state.hl_mark_price}) - external_perp_pxs = {} + external_perp_pxs = {} + + if self.enable_publish: + if self.enable_kms: + push_response = self.kms_signer.set_oracle( + dex=self.market_name, + oracle_pxs=oracle_pxs, + all_mark_pxs=mark_pxs, + external_perp_pxs=external_perp_pxs, + ) + else: + push_response = self.oracle_publisher_exchange.perp_deploy_set_oracle( + dex=self.market_name, + oracle_pxs=oracle_pxs, + all_mark_pxs=mark_pxs, + external_perp_pxs=external_perp_pxs, + ) - if self.enable_publish: - if self.enable_kms: - push_response = self.kms_signer.set_oracle( - dex=self.market_name, - oracle_pxs=oracle_pxs, - all_mark_pxs=mark_pxs, - external_perp_pxs=external_perp_pxs, - ) - else: - push_response = self.oracle_publisher_exchange.perp_deploy_set_oracle( - dex=self.market_name, - oracle_pxs=oracle_pxs, - all_mark_pxs=mark_pxs, - external_perp_pxs=external_perp_pxs, - ) - # TODO: Look at specific error responses and log/alert accordingly - logger.info("publish: push response: {}", push_response) + # TODO: Look at specific error responses and log/alert accordingly + logger.debug("publish: push response: {} {}", push_response, type(push_response)) + status = push_response.get("status", "") + if status == "ok": + self.metrics.successful_push_counter.add(1) + elif status == "err": + self.metrics.failed_push_counter.add(1) + logger.error("publish: publish error: {}", push_response) diff --git a/apps/hip-3-pusher/uv.lock b/apps/hip-3-pusher/uv.lock index 8a7bee3ba6..ec2845257e 100644 --- a/apps/hip-3-pusher/uv.lock +++ b/apps/hip-3-pusher/uv.lock @@ -365,7 +365,7 @@ wheels = [ [[package]] name = "hip-3-pusher" -version = "0.1.1" +version = "0.1.2" source = { virtual = "." } dependencies = [ { name = "asn1crypto" }, @@ -373,6 +373,9 @@ dependencies = [ { name = "cryptography" }, { name = "hyperliquid-python-sdk" }, { name = "loguru" }, + { name = "opentelemetry-exporter-prometheus" }, + { name = "opentelemetry-sdk" }, + { name = "prometheus-client" }, { name = "toml" }, { name = "websockets" }, ] @@ -384,6 +387,9 @@ requires-dist = [ { name = "cryptography", specifier = ">=45.0.7" }, { name = "hyperliquid-python-sdk", specifier = ">=0.19.0" }, { name = "loguru", specifier = ">=0.7.3" }, + { name = "opentelemetry-exporter-prometheus", specifier = ">=0.58b0" }, + { name = "opentelemetry-sdk", specifier = ">=1.37.0" }, + { name = "prometheus-client", specifier = ">=0.23.1" }, { name = "toml", specifier = ">=0.10.2" }, { name = "websockets", specifier = ">=15.0.1" }, ] @@ -413,6 +419,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + [[package]] name = "jmespath" version = "1.0.1" @@ -453,6 +471,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "importlib-metadata" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/04/05040d7ce33a907a2a02257e601992f0cdf11c73b33f13c4492bf6c3d6d5/opentelemetry_api-1.37.0.tar.gz", hash = "sha256:540735b120355bd5112738ea53621f8d5edb35ebcd6fe21ada3ab1c61d1cd9a7", size = 64923, upload-time = "2025-09-11T10:29:01.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/48/28ed9e55dcf2f453128df738210a980e09f4e468a456fa3c763dbc8be70a/opentelemetry_api-1.37.0-py3-none-any.whl", hash = "sha256:accf2024d3e89faec14302213bc39550ec0f4095d1cf5ca688e1bfb1c8612f47", size = 65732, upload-time = "2025-09-11T10:28:41.826Z" }, +] + +[[package]] +name = "opentelemetry-exporter-prometheus" +version = "0.58b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "prometheus-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/4c/e351559526ee35fa36d990d3455e81a5607c1fa3e544b599ad802f2481f8/opentelemetry_exporter_prometheus-0.58b0.tar.gz", hash = "sha256:70f2627b4bb82bac65a1fcf95f6e0dcce9e823dd47379ced854753a7e14dfc93", size = 14972, upload-time = "2025-09-11T10:29:05.513Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/e3/50e9cdc5a52c2ab19585dd69e668ec9fee0343fafc4bffa919ca79230a4f/opentelemetry_exporter_prometheus-0.58b0-py3-none-any.whl", hash = "sha256:02005033a7a108ab9f3000ff3aa49e2d03a8893b5bf3431322ffa246affbf951", size = 13016, upload-time = "2025-09-11T10:28:47.67Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/62/2e0ca80d7fe94f0b193135375da92c640d15fe81f636658d2acf373086bc/opentelemetry_sdk-1.37.0.tar.gz", hash = "sha256:cc8e089c10953ded765b5ab5669b198bbe0af1b3f89f1007d19acd32dc46dda5", size = 170404, upload-time = "2025-09-11T10:29:11.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/62/9f4ad6a54126fb00f7ed4bb5034964c6e4f00fcd5a905e115bd22707e20d/opentelemetry_sdk-1.37.0-py3-none-any.whl", hash = "sha256:8f3c3c22063e52475c5dbced7209495c2c16723d016d39287dfc215d1771257c", size = 131941, upload-time = "2025-09-11T10:28:57.83Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.58b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/1b/90701d91e6300d9f2fb352153fb1721ed99ed1f6ea14fa992c756016e63a/opentelemetry_semantic_conventions-0.58b0.tar.gz", hash = "sha256:6bd46f51264279c433755767bb44ad00f1c9e2367e1b42af563372c5a6fa0c25", size = 129867, upload-time = "2025-09-11T10:29:12.597Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/90/68152b7465f50285d3ce2481b3aec2f82822e3f52e5152eeeaf516bab841/opentelemetry_semantic_conventions-0.58b0-py3-none-any.whl", hash = "sha256:5564905ab1458b96684db1340232729fce3b5375a06e140e8904c78e4f815b28", size = 207954, upload-time = "2025-09-11T10:28:59.218Z" }, +] + [[package]] name = "parsimonious" version = "0.10.0" @@ -465,6 +537,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/0f/c8b64d9b54ea631fcad4e9e3c8dbe8c11bb32a623be94f22974c88e71eaf/parsimonious-0.10.0-py3-none-any.whl", hash = "sha256:982ab435fabe86519b57f6b35610aa4e4e977e9f02a14353edf4bbc75369fc0f", size = 48427, upload-time = "2022-09-03T17:01:13.814Z" }, ] +[[package]] +name = "prometheus-client" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/53/3edb5d68ecf6b38fcbcc1ad28391117d2a322d9a1a3eff04bfdb184d8c3b/prometheus_client-0.23.1.tar.gz", hash = "sha256:6ae8f9081eaaaf153a2e959d2e6c4f4fb57b12ef76c8c7980202f1e57b48b2ce", size = 80481, upload-time = "2025-09-18T20:47:25.043Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, +] + [[package]] name = "pycparser" version = "2.23" @@ -728,3 +809,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/b3/8f/705086c9d734d3b66 wheels = [ { url = "https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390", size = 4083, upload-time = "2024-12-07T15:28:26.465Z" }, ] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +]