Postgres config through environmental variables

README.md

@@ -74,3 +74,20 @@ This will generate `bots_config.default.json`, which you can feed into `evm_bots
 ```bash
 evm_bots.py bots_config.default.json
 ```
+
+## Data pipeline
+The data pipeline queries the running chain and exports data to a postgres database. The `infra` repostitory spins up a local postgres instance via Docker, and the data piepline will point to this by default. Optinally, you can also configure the backend database by specifying the following environmental variables (for example, in a `.env` file in the base of the repo):
+
+```bash
+POSTGRES_USER="admin"
+POSTGRES_PASSWORD="password"
+POSTGRES_DB="postgres_db"
+POSTGRES_HOST="localhost"
+POSTGRES_PORT=5432
+```
+
+The data script can be then ran using the following command:
+
+```bash
+python elfpy/data/acquire_data.py
+```

elfpy/data/acquire_data.py

@@ -4,6 +4,7 @@
 import logging
 import time
 
+from dotenv import load_dotenv
 from eth_typing import URI, BlockNumber
 from web3 import Web3
 
@@ -27,6 +28,7 @@ def main(
     # TODO: refactor this function, its waaay to big as indicated by these pylints
     # pylint: disable=too-many-locals
     # pylint: disable=too-many-statements
+
     # initialize the postgres session
     session = postgres.initialize_session()
     # get web3 provider
@@ -137,6 +139,10 @@ def main(
     # Look back limit for backfilling
     LOOKBACK_BLOCK_LIMIT = 1000
     SLEEP_AMOUNT = 1
+
+    # Get postgres env variables if exists
+    load_dotenv()
+
     log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True)
     main(
         CONTRACTS_URL,

elfpy/data/postgres.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import os
+from dataclasses import dataclass
+
 import pandas as pd
 import sqlalchemy
-from sqlalchemy import create_engine
+from sqlalchemy import URL, create_engine
 from sqlalchemy.orm import Session, sessionmaker
 
 from elfpy.data.db_schema import Base, PoolConfig, PoolInfo, Transaction, WalletInfo
@@ -14,12 +17,62 @@
 
 # replace the user, password, and db_name with credentials
 # TODO remove engine as global
-engine = create_engine("postgresql://admin:password@localhost:5432/postgres_db")
+
+
+@dataclass
+class PostgresConfig:
+    """The configuration dataclass for postgress connections"""
+
+    # default values for local postgres
+    # Matching environemnt variables to search for
+    # pylint: disable=invalid-name
+    POSTGRES_USER: str = "admin"
+    POSTGRES_PASSWORD: str = "password"
+    POSTGRES_DB: str = "postgres_db"
+    POSTGRES_HOST: str = "localhost"
+    POSTGRES_PORT: int = 5432
+
+
+def build_postgres_config() -> PostgresConfig:
+    """Build a PostgresConfig that looks for environmental variables
+    If env var exists, use that, otherwise, default
+    """
+    user = os.getenv("POSTGRES_USER")
+    password = os.getenv("POSTGRES_PASSWORD")
+    database = os.getenv("POSTGRES_DB")
+    host = os.getenv("POSTGRES_HOST")
+    port = os.getenv("POSTGRES_PORT")
+
+    arg_dict = {}
+    if user is not None:
+        arg_dict["POSTGRES_USER"] = user
+    if password is not None:
+        arg_dict["POSTGRES_PASSWORD"] = password
+    if database is not None:
+        arg_dict["POSTGRES_DB"] = database
+    if host is not None:
+        arg_dict["POSTGRES_HOST"] = host
+    if port is not None:
+        arg_dict["POSTGRES_PORT"] = int(port)
+
+    return PostgresConfig(**arg_dict)
 
 
 def initialize_session() -> Session:
     """Initialize the database if not already initialized"""
 
+    postgres_config = build_postgres_config()
+
+    url_object = URL.create(
+        drivername="postgresql",
+        username=postgres_config.POSTGRES_USER,
+        password=postgres_config.POSTGRES_PASSWORD,
+        host=postgres_config.POSTGRES_HOST,
+        port=postgres_config.POSTGRES_PORT,
+        database=postgres_config.POSTGRES_DB,
+    )
+    engine = create_engine(url_object)
+
     # create a configured "Session" class
     session_class = sessionmaker(bind=engine)
 

examples/hackweek_demo/run_demo.py

@@ -6,6 +6,7 @@
 
 import mplfinance as mpf
 import streamlit as st
+from dotenv import load_dotenv
 from extract_data_logs import get_combined_data
 from plot_fixed_rate import calc_fixed_rate, plot_fixed_rate
 from plot_ohlcv import calc_ohlcv, plot_ohlcv
@@ -44,6 +45,7 @@ def get_ticker(data):
 fig.set_tight_layout(True)  # type: ignore
 
 
+load_dotenv()
 session = postgres.initialize_session()
 
 if view_window is not None: