diff --git a/README.md b/README.md index f6096b451c..d3c60b57bc 100644 --- a/README.md +++ b/README.md @@ -74,3 +74,20 @@ This will generate `bots_config.default.json`, which you can feed into `evm_bots ```bash evm_bots.py bots_config.default.json ``` + +## Data pipeline +The data pipeline queries the running chain and exports data to a postgres database. The `infra` repostitory spins up a local postgres instance via Docker, and the data piepline will point to this by default. Optinally, you can also configure the backend database by specifying the following environmental variables (for example, in a `.env` file in the base of the repo): + +```bash +POSTGRES_USER="admin" +POSTGRES_PASSWORD="password" +POSTGRES_DB="postgres_db" +POSTGRES_HOST="localhost" +POSTGRES_PORT=5432 +``` + +The data script can be then ran using the following command: + +```bash +python elfpy/data/acquire_data.py +``` \ No newline at end of file diff --git a/elfpy/data/acquire_data.py b/elfpy/data/acquire_data.py index e82d56f98e..e204bc5e40 100644 --- a/elfpy/data/acquire_data.py +++ b/elfpy/data/acquire_data.py @@ -4,6 +4,7 @@ import logging import time +from dotenv import load_dotenv from eth_typing import URI, BlockNumber from web3 import Web3 @@ -27,6 +28,7 @@ def main( # TODO: refactor this function, its waaay to big as indicated by these pylints # pylint: disable=too-many-locals # pylint: disable=too-many-statements + # initialize the postgres session session = postgres.initialize_session() # get web3 provider @@ -137,6 +139,10 @@ def main( # Look back limit for backfilling LOOKBACK_BLOCK_LIMIT = 1000 SLEEP_AMOUNT = 1 + + # Get postgres env variables if exists + load_dotenv() + log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) main( CONTRACTS_URL, diff --git a/elfpy/data/postgres.py b/elfpy/data/postgres.py index 41e9ad4e8c..fd6fdbf4a5 100644 --- a/elfpy/data/postgres.py +++ b/elfpy/data/postgres.py @@ -2,9 +2,12 @@ from __future__ import annotations +import os +from dataclasses import dataclass + import pandas as pd import sqlalchemy -from sqlalchemy import create_engine +from sqlalchemy import URL, create_engine from sqlalchemy.orm import Session, sessionmaker from elfpy.data.db_schema import Base, PoolConfig, PoolInfo, Transaction, WalletInfo @@ -14,12 +17,62 @@ # replace the user, password, and db_name with credentials # TODO remove engine as global -engine = create_engine("postgresql://admin:password@localhost:5432/postgres_db") + + +@dataclass +class PostgresConfig: + """The configuration dataclass for postgress connections""" + + # default values for local postgres + # Matching environemnt variables to search for + # pylint: disable=invalid-name + POSTGRES_USER: str = "admin" + POSTGRES_PASSWORD: str = "password" + POSTGRES_DB: str = "postgres_db" + POSTGRES_HOST: str = "localhost" + POSTGRES_PORT: int = 5432 + + +def build_postgres_config() -> PostgresConfig: + """Build a PostgresConfig that looks for environmental variables + If env var exists, use that, otherwise, default + """ + user = os.getenv("POSTGRES_USER") + password = os.getenv("POSTGRES_PASSWORD") + database = os.getenv("POSTGRES_DB") + host = os.getenv("POSTGRES_HOST") + port = os.getenv("POSTGRES_PORT") + + arg_dict = {} + if user is not None: + arg_dict["POSTGRES_USER"] = user + if password is not None: + arg_dict["POSTGRES_PASSWORD"] = password + if database is not None: + arg_dict["POSTGRES_DB"] = database + if host is not None: + arg_dict["POSTGRES_HOST"] = host + if port is not None: + arg_dict["POSTGRES_PORT"] = int(port) + + return PostgresConfig(**arg_dict) def initialize_session() -> Session: """Initialize the database if not already initialized""" + postgres_config = build_postgres_config() + + url_object = URL.create( + drivername="postgresql", + username=postgres_config.POSTGRES_USER, + password=postgres_config.POSTGRES_PASSWORD, + host=postgres_config.POSTGRES_HOST, + port=postgres_config.POSTGRES_PORT, + database=postgres_config.POSTGRES_DB, + ) + engine = create_engine(url_object) + # create a configured "Session" class session_class = sessionmaker(bind=engine) diff --git a/examples/hackweek_demo/run_demo.py b/examples/hackweek_demo/run_demo.py index 02a224c4dc..393245875e 100644 --- a/examples/hackweek_demo/run_demo.py +++ b/examples/hackweek_demo/run_demo.py @@ -6,6 +6,7 @@ import mplfinance as mpf import streamlit as st +from dotenv import load_dotenv from extract_data_logs import get_combined_data from plot_fixed_rate import calc_fixed_rate, plot_fixed_rate from plot_ohlcv import calc_ohlcv, plot_ohlcv @@ -44,6 +45,7 @@ def get_ticker(data): fig.set_tight_layout(True) # type: ignore +load_dotenv() session = postgres.initialize_session() if view_window is not None: