diff --git a/README.md b/README.md index e99d8678c5..8329e98bc4 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Please refer to [BUILD.md](https://github.com/delvtech/elf-simulations/blob/main ## Testing +We deploy a local anvil chain to run system tests. Therefore, you must [install foundry](https://github.com/foundry-rs/foundry#installatio://github.com/foundry-rs/foundry#installation) as a prerequisite for running tests. + Testing is achieved with [py.test](https://docs.pytest.org/en/latest/contents.html). You can run all tests from the repository root directory by running `python -m pytest`, or you can pick a specific test in the `tests/` folder with `python -m pytest tests/{test_file.py}`. ## Coverage diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..8abc9633f2 --- /dev/null +++ b/conftest.py @@ -0,0 +1,35 @@ +# Hack to allow for vscode debugger to throw exception immediately +# instead of allowing pytest to catch the exception and report +# Based on https://stackoverflow.com/questions/62419998/how-can-i-get-pytest-to-not-catch-exceptions/62563106#62563106 + +# Use this in conjunction with the following launch.json configuration: +# { +# "name": "Debug Current Test", +# "type": "python", +# "request": "launch", +# "module": "pytest", +# "args": ["${file}"], +# "console": "integratedTerminal", +# "justMyCode": true, +# "env": { +# "_PYTEST_RAISE": "1" +# }, +# }, + +# Ignore docstrings for this file +# pylint: disable=missing-docstring + + +import os + +import pytest + +if os.getenv("_PYTEST_RAISE", "0") != "0": + + @pytest.hookimpl(tryfirst=True) + def pytest_exception_interact(call): + raise call.excinfo.value + + @pytest.hookimpl(tryfirst=True) + def pytest_internalerror(excinfo): + raise excinfo.value diff --git a/lib/agent0/agent0/__init__.py b/lib/agent0/agent0/__init__.py index 4dc59c2ffc..d2630e32b4 100644 --- a/lib/agent0/agent0/__init__.py +++ b/lib/agent0/agent0/__init__.py @@ -1,2 +1,7 @@ """Account key config and various helper functions""" -from .accounts_config import AccountKeyConfig, build_account_config_from_env, initialize_accounts +from .accounts_config import ( + AccountKeyConfig, + build_account_config_from_env, + build_account_key_config_from_agent_config, + initialize_accounts, +) diff --git a/lib/agent0/agent0/accounts_config.py b/lib/agent0/agent0/accounts_config.py index 0f98666f8d..6825e11378 100644 --- a/lib/agent0/agent0/accounts_config.py +++ b/lib/agent0/agent0/accounts_config.py @@ -58,13 +58,33 @@ def to_env_str(self) -> str: def initialize_accounts( - agent_config: list[AgentConfig], env_file: str | None = None, random_seed: int = 1, develop: bool = False + agent_config: list[AgentConfig], + env_file: str | None = None, + random_seed: int = 1, + develop: bool = False, ) -> AccountKeyConfig: """ Build or load an accounts environment file. If it doesn't exist, create it based on agent_config. (if develop is off, print instructions on adding in user private key and running script to fund agents). If it does exist, read it in and use it. + + Arguments + --------- + agent_config: list[AgentConfig] + The list of agent configs that define policies and arguments. + env_file: str | None + The path to the env file to write/load from. Defaults to `accounts.env`. + random_seed: int + Random seed to use for initializing budgets. + develop: bool + Flag for development mode. If False, will exit if env_file doesn't exist and print instructions + on how to fund bot. + + Returns + ------- + AccountKeyConfig + The account config object linked to the env file. """ # Default location if env_file is None: @@ -102,21 +122,23 @@ def initialize_accounts( def build_account_key_config_from_agent_config( - agent_configs: list[AgentConfig], random_seed: int, user_key: str | None = None + agent_configs: list[AgentConfig], random_seed: int = 1, user_key: str | None = None ) -> AccountKeyConfig: """Build an Account Config from a provided agent config. Arguments -------- + agent_config: list[AgentConfig] + The list of agent configs that define policies and arguments. + random_seed: int + The seed to initialize the random generator to pass for each bot user_key: str The provided user key to use - agent_configs: list[AgentConfig] - The provided agent configs Returns ------- - AccountConfig - Config settings required to connect to the eth node + AccountKeyConfig + The account config object linked to the env file. """ rng = np.random.default_rng(random_seed) agent_private_keys = [] @@ -151,6 +173,13 @@ def build_account_key_config_from_agent_config( def build_account_config_from_env(env_file: str | None = None, user_key: str | None = None) -> AccountKeyConfig: """Build an Account Config from environmental variables. + Arguments + -------- + env_file: str | None + The path to the env file to load from. Defaults to `accounts.env`. + user_key: str + The provided user key to use + Returns ------- AccountConfig diff --git a/lib/agent0/agent0/hyperdrive/exec/__init__.py b/lib/agent0/agent0/hyperdrive/exec/__init__.py index 3fc0ccda1d..90778a5c76 100644 --- a/lib/agent0/agent0/hyperdrive/exec/__init__.py +++ b/lib/agent0/agent0/hyperdrive/exec/__init__.py @@ -12,5 +12,5 @@ from .fund_agents import fund_agents from .get_agent_accounts import get_agent_accounts from .run_agents import run_agents -from .setup_experiment import get_web3_and_contracts, register_username, setup_experiment +from .setup_experiment import register_username, setup_experiment from .trade_loop import get_wait_for_new_block, trade_if_new_block diff --git a/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py b/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py index 78b72a0f95..702d3dbb60 100644 --- a/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py +++ b/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py @@ -7,9 +7,7 @@ from eth_account.account import Account from ethpy import EthConfig from ethpy.base import set_anvil_account_balance, smart_contract_transact -from ethpy.hyperdrive.addresses import HyperdriveAddresses - -from .setup_experiment import get_web3_and_contracts +from ethpy.hyperdrive import HyperdriveAddresses, get_web3_and_hyperdrive_contracts def create_and_fund_user_account( @@ -38,7 +36,7 @@ def create_and_fund_user_account( user_private_key = make_private_key(extra_entropy="FAKE USER") # argument value can be any str user_account = HyperdriveAgent(Account().from_key(user_private_key)) - web3, base_token_contract, _ = get_web3_and_contracts(eth_config, contract_addresses) + web3, base_token_contract, _ = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) eth_balance = sum((int(budget) for budget in account_key_config.AGENT_ETH_BUDGETS)) * 2 # double for good measure _ = set_anvil_account_balance(web3, user_account.address, eth_balance) diff --git a/lib/agent0/agent0/hyperdrive/exec/fund_agents.py b/lib/agent0/agent0/hyperdrive/exec/fund_agents.py index abcdf9005b..ba71e3df43 100644 --- a/lib/agent0/agent0/hyperdrive/exec/fund_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/fund_agents.py @@ -15,7 +15,7 @@ smart_contract_read, smart_contract_transact, ) -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses def fund_agents( diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index b4882eccef..157ea4e315 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -9,7 +9,7 @@ from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config -from ethpy.hyperdrive.addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url +from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_url from .create_and_fund_user_account import create_and_fund_user_account from .fund_agents import fund_agents @@ -26,7 +26,7 @@ def run_agents( account_key_config: AccountKeyConfig, develop: bool = False, eth_config: EthConfig | None = None, - override_addresses: HyperdriveAddresses | None = None, + contract_addresses: HyperdriveAddresses | None = None, ) -> None: """Entrypoint to run agents. @@ -43,24 +43,22 @@ def run_agents( eth_config: EthConfig | None Configuration for urls to the rpc and artifacts. If not set, will look for addresses in eth.env. - override_addresses: HyperdriveAddresses | None + contract_addresses: HyperdriveAddresses | None If set, will use these addresses instead of querying the artifact url defined in eth_config. """ - # Defaults to looking for eth_config env - if eth_config is None: - eth_config = build_eth_config() - # Set sane logging defaults to avoid spam from dependencies logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("web3").setLevel(logging.WARNING) warnings.filterwarnings("ignore", category=UserWarning, module="web3.contract.base_contract") - # Get addresses either from artifacts url defined in eth_config or from override_addresses - if override_addresses is not None: - contract_addresses = override_addresses - else: + # Defaults to looking for eth_config env + if eth_config is None: + eth_config = build_eth_config() + + # Get addresses either from artifacts url defined in eth_config or from contract_addresses + if contract_addresses is None: contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) if develop: # setup env automatically & fund the agents diff --git a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py index d01b3dbccb..1f6b3dc002 100644 --- a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py +++ b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py @@ -11,8 +11,7 @@ from agent0.hyperdrive.exec.crash_report import setup_hyperdrive_crash_report_logging from elfpy.utils import logs from ethpy import EthConfig -from ethpy.base import initialize_web3_with_http_provider, load_all_abis -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses, get_web3_and_hyperdrive_contracts from web3 import Web3 from web3.contract.contract import Contract @@ -65,7 +64,7 @@ def setup_experiment( log_format_string=environment_config.log_formatter, ) setup_hyperdrive_crash_report_logging() - web3, base_token_contract, hyperdrive_contract = get_web3_and_contracts(eth_config, contract_addresses) + web3, base_token_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) # load agent policies # rng is shared by the agents and can be accessed via `agent_accounts[idx].policy.rng` agent_accounts = get_agent_accounts( @@ -74,43 +73,6 @@ def setup_experiment( return web3, base_token_contract, hyperdrive_contract, agent_accounts -def get_web3_and_contracts( - eth_config: EthConfig, contract_addresses: HyperdriveAddresses -) -> tuple[Web3, Contract, Contract]: - """Get the web3 container and the ERC20Base and Hyperdrive contracts. - - Arguments - --------- - eth_config: EthConfig - Configuration for urls to the rpc and artifacts. - contract_addresses: HyperdriveAddresses - Configuration for defining various contract addresses. - - Returns - ------- - tuple[Web3, Contract, Contract] - A tuple containing: - - The web3 container - - The base token contract - - The hyperdrive contract - """ - # point to chain env - web3 = initialize_web3_with_http_provider(eth_config.RPC_URL, reset_provider=False) - # setup base contract interface - abis = load_all_abis(eth_config.ABI_DIR) - # set up the ERC20 contract for minting base tokens - # TODO is there a better way to pass in base and hyperdrive abi? - base_token_contract: Contract = web3.eth.contract( - abi=abis["ERC20Mintable"], address=web3.to_checksum_address(contract_addresses.base_token) - ) - # set up hyperdrive contract - hyperdrive_contract: Contract = web3.eth.contract( - abi=abis["IHyperdrive"], - address=web3.to_checksum_address(contract_addresses.mock_hyperdrive), - ) - return web3, base_token_contract, hyperdrive_contract - - def register_username(register_url: str, wallet_addrs: list[str], username: str) -> None: """Registers the username with the flask server. diff --git a/lib/agent0/agent0/test_fixtures/__init__.py b/lib/agent0/agent0/test_fixtures/__init__.py new file mode 100644 index 0000000000..d93fc97880 --- /dev/null +++ b/lib/agent0/agent0/test_fixtures/__init__.py @@ -0,0 +1,2 @@ +"""Test fixtures for agent0""" +from .cycle_trade_policy import AgentDoneException, cycle_trade_policy diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py new file mode 100644 index 0000000000..2efa71893e --- /dev/null +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -0,0 +1,156 @@ +"""Pytest fixture that creates an in memory db session and creates dummy db schemas""" +from __future__ import annotations + +from typing import Type + +import pytest +from agent0.hyperdrive.agents import HyperdriveWallet +from agent0.hyperdrive.policies import HyperdrivePolicy +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState +from elfpy.types import MarketType, Trade +from fixedpointmath import FixedPoint +from numpy.random._generator import Generator as NumpyGenerator + + +class AgentDoneException(Exception): + """Custom exception for signaling the bot is done""" + + +# Build custom policy +# Simple agent, opens a set of all trades for a fixed amount and closes them after +class CycleTradesPolicy(HyperdrivePolicy): + """A agent that simply cycles through all trades""" + + # Using default parameters + def __init__( + self, + budget: FixedPoint, + rng: NumpyGenerator | None = None, + slippage_tolerance: FixedPoint | None = None, + ): + # We want to do a sequence of trades one at a time, so we keep an internal counter based on + # how many times `action` has been called. + self.counter = 0 + super().__init__(budget, rng, slippage_tolerance) + + def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMarketAction]]: + """This agent simply opens all trades for a fixed amount and closes them after, one at a time""" + action_list = [] + if self.counter == 0: + # Add liquidity + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.ADD_LIQUIDITY, + trade_amount=FixedPoint(scaled_value=int(11111e18)), + wallet=wallet, + ), + ) + ) + elif self.counter == 1: + # Open Long + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_LONG, + trade_amount=FixedPoint(scaled_value=int(22222e18)), + wallet=wallet, + ), + ) + ) + elif self.counter == 2: + # Open Short + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_SHORT, + trade_amount=FixedPoint(scaled_value=int(33333e18)), + wallet=wallet, + ), + ) + ) + elif self.counter == 3: + # Remove All Liquidity + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.REMOVE_LIQUIDITY, + trade_amount=wallet.lp_tokens, + wallet=wallet, + ), + ) + ) + elif self.counter == 4: + # Close All Longs + assert len(wallet.longs) == 1 + for long_time, long in wallet.longs.items(): + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.CLOSE_LONG, + trade_amount=long.balance, + wallet=wallet, + # TODO is this actually maturity time? Not mint time? + mint_time=long_time, + ), + ) + ) + elif self.counter == 5: + # Close All Shorts + assert len(wallet.shorts) == 1 + for short_time, short in wallet.shorts.items(): + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.CLOSE_SHORT, + trade_amount=short.balance, + wallet=wallet, + # TODO is this actually maturity time? Not mint time? + mint_time=short_time, + ), + ) + ) + elif self.counter == 6: + # Redeem all withdrawal shares + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.REDEEM_WITHDRAW_SHARE, + trade_amount=wallet.withdraw_shares, + wallet=wallet, + ), + ) + ) + elif self.counter == 7: + # One more dummy trade to ensure the previous trades get into the db + # TODO test if we can remove this eventually by allowing acquire_data to look at + # current block + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_LONG, + trade_amount=FixedPoint(scaled_value=int(1e18)), + wallet=wallet, + ), + ) + ) + else: + # We want this bot to exit and crash after it's done the trades it needs to do + raise AgentDoneException("Bot done") + self.counter += 1 + return action_list + + +@pytest.fixture(scope="function") +def cycle_trade_policy() -> Type[CycleTradesPolicy]: + """Test fixture to build a policy that cycles through all trades""" + return CycleTradesPolicy diff --git a/lib/agent0/bin/fund_agents_from_user_key.py b/lib/agent0/bin/fund_agents_from_user_key.py index 02b238b1d9..f86bf187ae 100644 --- a/lib/agent0/bin/fund_agents_from_user_key.py +++ b/lib/agent0/bin/fund_agents_from_user_key.py @@ -9,7 +9,7 @@ from agent0.hyperdrive.exec import fund_agents from eth_account.account import Account from ethpy import build_eth_config -from ethpy.hyperdrive.addresses import fetch_hyperdrive_address_from_url +from ethpy.hyperdrive import fetch_hyperdrive_address_from_url if __name__ == "__main__": parser = argparse.ArgumentParser( diff --git a/lib/agent0/examples/example_agent.py b/lib/agent0/examples/example_agent.py index 8a51d8df00..9987b561ee 100644 --- a/lib/agent0/examples/example_agent.py +++ b/lib/agent0/examples/example_agent.py @@ -24,6 +24,9 @@ # Build custom policy # Simple agent, opens a set of all trades for a fixed amount and closes them after +# TODO this bot is almost identical to the one defined in test_fixtures for system tests +# On one hand, this bot is nice for an example since it shows all trades +# On the other, duplicated code between the two bots class CycleTradesPolicy(HyperdrivePolicy): """An agent that simply cycles through all trades""" @@ -109,7 +112,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 4: + elif self.counter == 5: # Close All Shorts assert len(wallet.shorts) == 1 for short_time, short in wallet.shorts.items(): @@ -125,7 +128,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 5: + elif self.counter == 6: # Redeem all withdrawal shares action_list.append( Trade( @@ -137,7 +140,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 6: + elif self.counter == 7: # One more dummy trade to ensure the previous trades get into the db # TODO test if we can remove this eventually action_list.append( diff --git a/lib/chainsync/bin/run_chainsync.py b/lib/chainsync/bin/run_chainsync.py new file mode 100644 index 0000000000..b004936ce1 --- /dev/null +++ b/lib/chainsync/bin/run_chainsync.py @@ -0,0 +1,9 @@ +"""Script to format on-chain hyperdrive pool, config, and transaction data post-processing.""" +from __future__ import annotations + +from chainsync.exec import acquire_data +from elfpy.utils import logs as log_utils + +if __name__ == "__main__": + log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) + acquire_data() diff --git a/lib/chainsync/bin/run_hyperdrive_dashboard.py b/lib/chainsync/bin/run_hyperdrive_dashboard.py index f49e39af59..93d9ab16c6 100644 --- a/lib/chainsync/bin/run_hyperdrive_dashboard.py +++ b/lib/chainsync/bin/run_hyperdrive_dashboard.py @@ -34,9 +34,6 @@ # pool config data is static, so just read once config_data = get_pool_config(session, coerce_float=False) -# TODO fix input invTimeStretch to be unscaled in ingestion into postgres -config_data["invTimeStretch"] = config_data["invTimeStretch"] / 10**18 - config_data = config_data.iloc[0] diff --git a/lib/chainsync/chainsync/db/hyperdrive/convert_data.py b/lib/chainsync/chainsync/db/hyperdrive/convert_data.py index 30c43db1f6..caa19effc2 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/convert_data.py +++ b/lib/chainsync/chainsync/db/hyperdrive/convert_data.py @@ -565,7 +565,7 @@ def _build_hyperdrive_transaction_object( "transactionHash": transaction_dict["hash"], "txn_to": transaction_dict["to"], "txn_from": transaction_dict["from"], - "gasUsed": receipt["gasUsed"], + "gasUsed": _convert_scaled_value_to_decimal(receipt["gasUsed"]), } # Input solidity methods and parameters # TODO can the input field ever be empty or not exist? diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface.py b/lib/chainsync/chainsync/db/hyperdrive/interface.py index 358ee74b70..c64b124790 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface.py @@ -205,6 +205,8 @@ def get_pool_info( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -230,7 +232,9 @@ def get_pool_info( return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_transactions(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_transactions( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> pd.DataFrame: """Get all transactions and returns as a pandas dataframe. Arguments @@ -243,6 +247,8 @@ def get_transactions(session: Session, start_block: int | None = None, end_block end_block : int | None The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -262,10 +268,12 @@ def get_transactions(session: Session, start_block: int | None = None, end_block if end_block is not None: query = query.filter(HyperdriveTransaction.blockNumber < end_block) - return pd.read_sql(query.statement, con=session.connection()).set_index("blockNumber") + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_checkpoint_info(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_checkpoint_info( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> pd.DataFrame: """Get all info associated with a given checkpoint. This includes @@ -283,6 +291,8 @@ def get_checkpoint_info(session: Session, start_block: int | None = None, end_bl end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -305,10 +315,12 @@ def get_checkpoint_info(session: Session, start_block: int | None = None, end_bl # Always sort by time in order query = query.order_by(CheckpointInfo.timestamp) - return pd.read_sql(query.statement, con=session.connection()).set_index("blockNumber") + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_all_wallet_info(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_all_wallet_info( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float: bool = True +) -> pd.DataFrame: """Get all of the wallet_info data in history and returns as a pandas dataframe. Arguments @@ -321,6 +333,8 @@ def get_all_wallet_info(session: Session, start_block: int | None = None, end_bl end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -340,16 +354,18 @@ def get_all_wallet_info(session: Session, start_block: int | None = None, end_bl if end_block is not None: query = query.filter(WalletInfo.blockNumber < end_block) - return pd.read_sql(query.statement, con=session.connection()) + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) -def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: +def get_wallet_info_history(session: Session, coerce_float=True) -> dict[str, pd.DataFrame]: """Get the history of all wallet info over block time. Arguments --------- session : Session The initialized session object + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -359,8 +375,8 @@ def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: token the address has at that block number, plus a timestamp and the share price of the block """ # Get data - all_wallet_info = get_all_wallet_info(session) - pool_info_lookup = get_pool_info(session)[["timestamp", "sharePrice"]] + all_wallet_info = get_all_wallet_info(session, coerce_float=coerce_float) + pool_info_lookup = get_pool_info(session, coerce_float=coerce_float)[["timestamp", "sharePrice"]] # Pivot tokenType to columns, keeping walletAddress and blockNumber all_wallet_info = all_wallet_info.pivot( @@ -389,7 +405,7 @@ def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: def get_current_wallet_info( - session: Session, start_block: int | None = None, end_block: int | None = None + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float: bool = True ) -> pd.DataFrame: """Get the balance of a wallet and a given end_block. @@ -409,13 +425,17 @@ def get_current_wallet_info( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- DataFrame A DataFrame that consists of the queried wallet info data """ - all_wallet_info = get_all_wallet_info(session, start_block=start_block, end_block=end_block) + all_wallet_info = get_all_wallet_info( + session, start_block=start_block, end_block=end_block, coerce_float=coerce_float + ) # Get last entry in the table of each wallet address and token type # This should always return a dataframe # Pandas doesn't play nice with types @@ -458,6 +478,8 @@ def get_wallet_deltas( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -480,7 +502,9 @@ def get_wallet_deltas( return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) -def get_all_traders(session: Session, start_block: int | None = None, end_block: int | None = None) -> list[str]: +def get_all_traders( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> list[str]: """Get the list of all traders from the WalletInfo table. Arguments @@ -493,6 +517,8 @@ def get_all_traders(session: Session, start_block: int | None = None, end_block: end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -515,12 +541,14 @@ def get_all_traders(session: Session, start_block: int | None = None, end_block: return [] query = query.distinct() - results = pd.read_sql(query.statement, con=session.connection()) + results = pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) return results["walletAddress"].to_list() -def get_agent_positions(session: Session, filter_addr: list[str] | None = None) -> dict[str, AgentPosition]: +def get_agent_positions( + session: Session, filter_addr: list[str] | None = None, coerce_float: bool = True +) -> dict[str, AgentPosition]: """Create an AgentPosition for each agent in the wallet history. Arguments @@ -529,6 +557,8 @@ def get_agent_positions(session: Session, filter_addr: list[str] | None = None) The initialized session object filter_addr : list[str] | None Only return these addresses. Returns all if None + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -536,9 +566,11 @@ def get_agent_positions(session: Session, filter_addr: list[str] | None = None) Returns a dictionary keyed by wallet address, value of an agent's position """ if filter_addr is None: - return {agent: AgentPosition(wallet) for agent, wallet in get_wallet_info_history(session).items()} + return { + agent: AgentPosition(wallet) for agent, wallet in get_wallet_info_history(session, coerce_float).items() + } return { agent: AgentPosition(wallet) - for agent, wallet in get_wallet_info_history(session).items() + for agent, wallet in get_wallet_info_history(session, coerce_float).items() if agent in filter_addr } diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py index ea2464261d..6c03599a06 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py @@ -149,43 +149,49 @@ def test_get_pool_config(self, db_session): pool_config_1 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config_1, db_session) - pool_config_df_1 = get_pool_config(db_session, coerce_float=False) + pool_config_df_1 = get_pool_config(db_session) assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type + # Internally, they store Numeric types as floats, hence we see rounding errors in testing + # This does not happen in postgres, where these values match exactly. + # https://github.com/delvtech/elf-simulations/issues/836 + np.testing.assert_array_equal(pool_config_df_1["initialSharePrice"], np.array([3.2])) pool_config_2 = PoolConfig(contractAddress="1", initialSharePrice=Decimal("3.4")) add_pool_config(pool_config_2, db_session) - pool_config_df_2 = get_pool_config(db_session, coerce_float=False) + pool_config_df_2 = get_pool_config(db_session) assert len(pool_config_df_2) == 2 - np.testing.assert_array_equal(pool_config_df_2["initialSharePrice"], np.array([Decimal("3.2"), Decimal("3.4")])) + np.testing.assert_array_equal(pool_config_df_2["initialSharePrice"], np.array([3.2, 3.4])) def test_primary_id_query_pool_config(self, db_session): """Testing retrieval of pool config via interface""" pool_config = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config, db_session) - pool_config_df_1 = get_pool_config(db_session, contract_address="0", coerce_float=False) + pool_config_df_1 = get_pool_config(db_session, contract_address="0") assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_1.loc[0, "initialSharePrice"] == 3.2 - pool_config_df_2 = get_pool_config(db_session, contract_address="1", coerce_float=False) + pool_config_df_2 = get_pool_config(db_session, contract_address="1") assert len(pool_config_df_2) == 0 def test_pool_config_verify(self, db_session): """Testing retrieval of pool config via interface""" pool_config_1 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config_1, db_session) - pool_config_df_1 = get_pool_config(db_session, coerce_float=False) + pool_config_df_1 = get_pool_config(db_session) assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_1.loc[0, "initialSharePrice"] == 3.2 # Nothing should happen if we give the same pool_config - pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) + # TODO Below is a hack due to sqlite not having numerics + # We explicitly print 18 spots after floating point to match rounding error in sqlite + pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal(f"{3.2:.18f}")) add_pool_config(pool_config_2, db_session) - pool_config_df_2 = get_pool_config(db_session, coerce_float=False) + pool_config_df_2 = get_pool_config(db_session) assert len(pool_config_df_2) == 1 - assert pool_config_df_2.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_2.loc[0, "initialSharePrice"] == 3.2 # If we try to add another pool config with a different value, should throw a ValueError pool_config_3 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.4")) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index 148463b338..3cbf57f437 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -10,6 +10,13 @@ # pylint: disable=invalid-name +# Postgres numeric type that matches fixedpoint +# Precision here indicates the total number of significant digits to store, +# while scale indicates the number of digits to the right of the decimal +# The high precision doesn't actually allocate memory in postgres, as numeric is variable size +# https://stackoverflow.com/questions/40686571/performance-of-numeric-type-with-high-precisions-and-scales-in-postgresql +FIXED_NUMERIC = Numeric(precision=1000, scale=18) + class PoolConfig(Base): """Table/dataclass schema for pool config.""" @@ -18,20 +25,20 @@ class PoolConfig(Base): contractAddress: Mapped[str] = mapped_column(String, primary_key=True) baseToken: Mapped[Union[str, None]] = mapped_column(String, default=None) - initialSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - minimumShareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + initialSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + minimumShareReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) positionDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None) checkpointDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - timeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + timeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) governance: Mapped[Union[str, None]] = mapped_column(String, default=None) feeCollector: Mapped[Union[str, None]] = mapped_column(String, default=None) - curveFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - flatFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - governanceFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - oracleSize: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + curveFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + flatFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + governanceFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + oracleSize: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None) + invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - termLength: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) class CheckpointInfo(Base): @@ -41,9 +48,9 @@ class CheckpointInfo(Base): blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True) timestamp: Mapped[datetime] = mapped_column(DateTime, index=True) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) class PoolInfo(Base): @@ -56,19 +63,19 @@ class PoolInfo(Base): blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True) timestamp: Mapped[datetime] = mapped_column(DateTime, index=True) - shareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - bondReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - lpTotalSupply: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - lpSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - withdrawalSharesReadyToWithdraw: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - withdrawalSharesProceeds: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - totalSupplyWithdrawalShares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + shareReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + bondReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + lpTotalSupply: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + lpSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + withdrawalSharesReadyToWithdraw: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + withdrawalSharesProceeds: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + totalSupplyWithdrawalShares: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # TODO: Rename this to something more accurate to what is happening, e.g. HyperdriveTransactions @@ -90,9 +97,9 @@ class WalletInfo(Base): baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None) # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) - tokenValue: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), default=None) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + tokenValue: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # TODO: either make a more general TokenDelta, or rename this to HyperdriveDelta @@ -114,8 +121,8 @@ class WalletDelta(Base): baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None) # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) - delta: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - maturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + delta: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) class HyperdriveTransaction(Base): @@ -142,7 +149,7 @@ class HyperdriveTransaction(Base): # Almost always from wallet address to smart contract address txn_to: Mapped[Union[str, None]] = mapped_column(String, default=None) txn_from: Mapped[Union[str, None]] = mapped_column(String, default=None) - gasUsed: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + gasUsed: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) #### Fields from solidity function calls #### # These fields map solidity function calls and their corresponding arguments @@ -150,25 +157,25 @@ class HyperdriveTransaction(Base): input_method: Mapped[Union[str, None]] = mapped_column(String, default=None) # Method: initialize - input_params_contribution: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_apr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_contribution: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_apr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) input_params_destination: Mapped[Union[str, None]] = mapped_column(String, default=None) input_params_asUnderlying: Mapped[Union[bool, None]] = mapped_column(Boolean, default=None) # Method: openLong - input_params_baseAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_minOutput: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_baseAmount: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_minOutput: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying # Method: openShort - input_params_bondAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_maxDeposit: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_bondAmount: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_maxDeposit: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying # Method: closeLong - input_params_maturityTime: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + input_params_maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) # input_params_bondAmount # input_params_minOutput # input_params_destination @@ -183,13 +190,13 @@ class HyperdriveTransaction(Base): # Method: addLiquidity # input_params_contribution - input_params_minApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_maxApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_minApr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_maxApr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying # Method: removeLiquidity - input_params_shares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_shares: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_minOutput # input_params_destination # input_params_asUnderlying @@ -201,12 +208,14 @@ class HyperdriveTransaction(Base): # args_owner # args_spender # args_id - event_value: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + event_value: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) event_operator: Mapped[Union[str, None]] = mapped_column(String, default=None) - event_id: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + event_id: Mapped[Union[int, None]] = mapped_column( + Numeric, default=None + ) # Integer too small here to store event_id, so we use Numeric instead # Fields calculated from base event_prefix: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - event_maturity_time: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + event_maturity_time: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) # Fields not used by postprocessing diff --git a/lib/chainsync/chainsync/exec/__init__.py b/lib/chainsync/chainsync/exec/__init__.py new file mode 100644 index 0000000000..d79bc21bb0 --- /dev/null +++ b/lib/chainsync/chainsync/exec/__init__.py @@ -0,0 +1,2 @@ +"""Execution functions for chainsync""" +from .acquire_data import acquire_data diff --git a/lib/chainsync/bin/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py similarity index 59% rename from lib/chainsync/bin/acquire_data.py rename to lib/chainsync/chainsync/exec/acquire_data.py index b766897303..a338ae7621 100644 --- a/lib/chainsync/bin/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -11,59 +11,64 @@ get_latest_block_number_from_pool_info_table, init_data_chain_to_db, ) -from elfpy.utils import logs as log_utils -from eth_typing import URI, BlockNumber -from eth_utils import address -from ethpy import build_eth_config -from ethpy.base import initialize_web3_with_http_provider, load_all_abis -from ethpy.hyperdrive import fetch_hyperdrive_address_from_url -from ethpy.hyperdrive.interface import get_hyperdrive_contract -from web3 import Web3 -from web3.contract.contract import Contract +from eth_typing import BlockNumber +from ethpy import EthConfig, build_eth_config +from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_url, get_web3_and_hyperdrive_contracts +from sqlalchemy.orm import Session _SLEEP_AMOUNT = 1 -def main( - artifacts_url: str, - rpc_url: URI | str, - abi_dir: str, - start_block: int, - lookback_block_limit: int, +# Lots of arguments +# pylint: disable=too-many-arguments +def acquire_data( + start_block: int = 0, + lookback_block_limit: int = 10000, + eth_config: EthConfig | None = None, + db_session: Session | None = None, + contract_addresses: HyperdriveAddresses | None = None, + exit_on_catch_up: bool = False, ): """Execute the data acquisition pipeline. Arguments --------- - artifacts_url: str - The url of the artifacts server from which we get addresses. - rpc_url: URI | str - The url to the ethereum node - abi_dir : str - The path to the abi directory start_block : int The starting block to filter the query on lookback_block_limit : int The maximum number of blocks to look back when filling in missing data + eth_config: EthConfig | None + Configuration for urls to the rpc and artifacts. If not set, will look for addresses + in eth.env. + db_session: Session | None + Session object for connecting to db. If None, will initialize a new session based on + postgres.env. + contract_addresses: HyperdriveAddresses | None + If set, will use these addresses instead of querying the artifact url + defined in eth_config. + exit_on_catch_up: bool + If True, will exit after catching up to current block """ ## Initialization + # eth config + if eth_config is None: + # Load parameters from env vars if they exist + eth_config = build_eth_config() + # postgres session - session = initialize_session() - # web3 provider - web3: Web3 = initialize_web3_with_http_provider(rpc_url, request_kwargs={"timeout": 60}) - # send a request to the local server to fetch the deployed contract addresses and - # all Hyperdrive contract addresses from the server response - addresses = fetch_hyperdrive_address_from_url(os.path.join(artifacts_url, "addresses.json")) - abis = load_all_abis(abi_dir) - # Contracts - hyperdrive_contract = get_hyperdrive_contract(web3, abis, addresses) - base_contract: Contract = web3.eth.contract( - address=address.to_checksum_address(addresses.base_token), abi=abis["ERC20Mintable"] - ) + if db_session is None: + db_session = initialize_session() + + # Get addresses either from artifacts url defined in eth_config or from contract_addresses + if contract_addresses is None: + contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) + + # Get web3 and contracts + web3, base_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) ## Get starting point for restarts # Get last entry of pool info in db - data_latest_block_number = get_latest_block_number_from_pool_info_table(session) + data_latest_block_number = get_latest_block_number_from_pool_info_table(db_session) # Using max of latest block in database or specified start block block_number: BlockNumber = BlockNumber(max(start_block, data_latest_block_number)) # Make sure to not grab current block, as the current block is subject to change @@ -76,11 +81,11 @@ def main( logging.warning("Starting block is past lookback block limit, starting at block %s", block_number) # Collect initial data - init_data_chain_to_db(hyperdrive_contract, session) + init_data_chain_to_db(hyperdrive_contract, db_session) # This if statement executes only on initial run (based on data_latest_block_number check), # and if the chain has executed until start_block (based on latest_mined_block check) if data_latest_block_number < block_number < latest_mined_block: - data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, session) + data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, db_session) # Main data loop # monitor for new blocks & add pool info per block @@ -90,6 +95,8 @@ def main( # Only execute if we are on a new block if latest_mined_block <= block_number: time.sleep(_SLEEP_AMOUNT) + if exit_on_catch_up: + break continue # Backfilling for blocks that need updating for block_int in range(block_number + 1, latest_mined_block + 1): @@ -103,24 +110,5 @@ def main( latest_mined_block, ) continue - data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, session) + data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, db_session) time.sleep(_SLEEP_AMOUNT) - - -if __name__ == "__main__": - # setup constants - START_BLOCK = 0 - # Look back limit for backfilling - LOOKBACK_BLOCK_LIMIT = 100000 - - # Load parameters from env vars if they exist - config = build_eth_config() - - log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) - main( - config.ARTIFACTS_URL, - config.RPC_URL, - config.ABI_DIR, - START_BLOCK, - LOOKBACK_BLOCK_LIMIT, - ) diff --git a/lib/ethpy/ethpy/hyperdrive/__init__.py b/lib/ethpy/ethpy/hyperdrive/__init__.py index 9053dbf18b..47256d3e00 100644 --- a/lib/ethpy/ethpy/hyperdrive/__init__.py +++ b/lib/ethpy/ethpy/hyperdrive/__init__.py @@ -2,6 +2,7 @@ from .addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url from .assets import AssetIdPrefix, decode_asset_id, encode_asset_id from .errors import HyperdriveErrors, lookup_hyperdrive_error_selector +from .get_web3_and_hyperdrive_contracts import get_web3_and_hyperdrive_contracts from .interface import ( get_hyperdrive_checkpoint_info, get_hyperdrive_config, diff --git a/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py b/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py new file mode 100644 index 0000000000..5036650e21 --- /dev/null +++ b/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py @@ -0,0 +1,53 @@ +"""Helper function for getting web3 and contracts.""" +from __future__ import annotations + +import os + +from ethpy import EthConfig +from ethpy.base import initialize_web3_with_http_provider, load_all_abis +from web3 import Web3 +from web3.contract.contract import Contract + +from .addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url + + +def get_web3_and_hyperdrive_contracts( + eth_config: EthConfig, contract_addresses: HyperdriveAddresses | None = None +) -> tuple[Web3, Contract, Contract]: + """Get the web3 container and the ERC20Base and Hyperdrive contracts. + + Arguments + --------- + eth_config: EthConfig + Configuration for urls to the rpc and artifacts. + contract_addresses: HyperdriveAddresses | None + Configuration for defining various contract addresses. + Will query eth_config artifacts for addresses by default + + Returns + ------- + tuple[Web3, Contract, Contract] + A tuple containing: + - The web3 container + - The base token contract + - The hyperdrive contract + """ + # Initialize contract addresses if none + if contract_addresses is None: + contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) + + # point to chain env + web3 = initialize_web3_with_http_provider(eth_config.RPC_URL, reset_provider=False) + # setup base contract interface + abis = load_all_abis(eth_config.ABI_DIR) + # set up the ERC20 contract for minting base tokens + # TODO is there a better way to pass in base and hyperdrive abi? + base_token_contract: Contract = web3.eth.contract( + abi=abis["ERC20Mintable"], address=web3.to_checksum_address(contract_addresses.base_token) + ) + # set up hyperdrive contract + hyperdrive_contract: Contract = web3.eth.contract( + abi=abis["IHyperdrive"], + address=web3.to_checksum_address(contract_addresses.mock_hyperdrive), + ) + return web3, base_token_contract, hyperdrive_contract diff --git a/lib/ethpy/ethpy/hyperdrive/interface.py b/lib/ethpy/ethpy/hyperdrive/interface.py index d573b4dac0..b98f6dae97 100644 --- a/lib/ethpy/ethpy/hyperdrive/interface.py +++ b/lib/ethpy/ethpy/hyperdrive/interface.py @@ -113,12 +113,11 @@ def get_hyperdrive_config(hyperdrive_contract: Contract) -> dict[str, Any]: pool_config["minimumShareReserves"] = FixedPoint(scaled_value=hyperdrive_config["minimumShareReserves"]) pool_config["positionDuration"] = hyperdrive_config["positionDuration"] pool_config["checkpointDuration"] = hyperdrive_config["checkpointDuration"] - # Ok so, the contracts store the time stretch constant in an inverted manner from the python. + # TODO Ok so, the contracts store the time stretch constant in an inverted manner from the python. # In order to not break the world, we save the contract version as 'invTimeStretch' and invert - # that to get the python version 'timeStretch' - # TODO invTimeStretch should be in fixed point notation - pool_config["invTimeStretch"] = hyperdrive_config["timeStretch"] - pool_config["timeStretch"] = FixedPoint(1) / FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) + # that to get the python version 'timeStretch'. Fix this issue to match solidity + pool_config["invTimeStretch"] = FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) + pool_config["timeStretch"] = FixedPoint(1) / pool_config["invTimeStretch"] pool_config["governance"] = hyperdrive_config["governance"] pool_config["feeCollector"] = hyperdrive_config["feeCollector"] curve_fee, flat_fee, governance_fee = hyperdrive_config["fees"] diff --git a/lib/ethpy/ethpy/test_fixtures/__init__.py b/lib/ethpy/ethpy/test_fixtures/__init__.py index a09453224f..3465d2a585 100644 --- a/lib/ethpy/ethpy/test_fixtures/__init__.py +++ b/lib/ethpy/ethpy/test_fixtures/__init__.py @@ -1,2 +1,2 @@ """Test fixtures for ethpy""" -from .local_chain import hyperdrive_contract_address, local_chain +from .local_chain import local_chain, local_hyperdrive_chain diff --git a/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py b/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py index 7a3cbd9d80..571302dc6a 100644 --- a/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py +++ b/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py @@ -84,10 +84,10 @@ def deploy_hyperdrive_factory(rpc_url: str, deploy_account: LocalAccount) -> tup initial_variable_rate = int(0.05e18) curve_fee = int(0.1e18) # 10% flat_fee = int(0.0005e18) # 0.05% - governance_fee = int(0.15e18) # 0.15% + governance_fee = int(0.15e18) # 15% max_curve_fee = int(0.3e18) # 30% max_flat_fee = int(0.0015e18) # 0.15% - max_governance_fee = int(0.30e18) # 0.30% + max_governance_fee = int(0.30e18) # 30% # Configuration settings abi_folder = "packages/hyperdrive/src/abis/" diff --git a/lib/ethpy/ethpy/test_fixtures/local_chain.py b/lib/ethpy/ethpy/test_fixtures/local_chain.py index 29c68c91d4..510ea8fbe6 100644 --- a/lib/ethpy/ethpy/test_fixtures/local_chain.py +++ b/lib/ethpy/ethpy/test_fixtures/local_chain.py @@ -5,6 +5,8 @@ import pytest from ethpy.base import initialize_web3_with_http_provider +from ethpy.hyperdrive import HyperdriveAddresses +from web3 import Web3 from .deploy_hyperdrive import deploy_and_initialize_hyperdrive, deploy_hyperdrive_factory, initialize_deploy_account @@ -14,8 +16,12 @@ @pytest.fixture(scope="function") def local_chain() -> Generator[str, Any, Any]: - """Launches a local anvil chain for testing. - Returns the chain url. + """Launches a local anvil chain for testing. Kills the anvil chain after. + + Returns + ------- + Generator[str, Any, Any] + Yields the local anvil chain url """ anvil_port = 9999 host = "127.0.0.1" # localhost @@ -31,7 +37,7 @@ def local_chain() -> Generator[str, Any, Any]: local_chain_ = "http://" + host + ":" + str(anvil_port) - # Hack, wait for anvil chain to initialize + # TODO Hack, wait for anvil chain to initialize time.sleep(3) yield local_chain_ @@ -41,12 +47,39 @@ def local_chain() -> Generator[str, Any, Any]: @pytest.fixture(scope="function") -def hyperdrive_contract_address(local_chain: str) -> str: +def local_hyperdrive_chain(local_chain: str) -> dict: """Initializes hyperdrive on a local anvil chain for testing. Returns the hyperdrive contract address. + Arguments + --------- + local_chain: str + The `local_chain` test fixture that binds to the local anvil chain rpc url + + Returns + ------- + dict + A dictionary with the following key - value fields: + + "web3": Web3 + web3 provider object + "deploy_account": LocalAccount + The local account that deploys and initializes hyperdrive + "hyperdrive_contract_addresses": HyperdriveAddresses + The hyperdrive contract addresses """ web3 = initialize_web3_with_http_provider(local_chain, reset_provider=False) account = initialize_deploy_account(web3) base_token_contract, factory_contract = deploy_hyperdrive_factory(local_chain, account) - return deploy_and_initialize_hyperdrive(web3, base_token_contract, factory_contract, account) + hyperdrive_addr = deploy_and_initialize_hyperdrive(web3, base_token_contract, factory_contract, account) + + return { + "web3": web3, + "deploy_account": account, + "hyperdrive_contract_addresses": HyperdriveAddresses( + base_token=Web3.to_checksum_address(base_token_contract.address), + hyperdrive_factory=Web3.to_checksum_address(factory_contract.address), + mock_hyperdrive=Web3.to_checksum_address(hyperdrive_addr), + mock_hyperdrive_math="not_used", + ), + } diff --git a/tests/system_test.py b/tests/system_test.py index 4490404ce7..08a35358ee 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -1,17 +1,377 @@ """System test for end to end testing of elf-simulations""" -from chainsync.test_fixtures import db_session # pylint: disable=unused-import -from ethpy.test_fixtures import hyperdrive_contract_address, local_chain # pylint: disable=unused-import +import logging +from decimal import Decimal +from typing import Type + +import numpy as np +import pandas as pd +from agent0 import build_account_key_config_from_agent_config +from agent0.base.config import AgentConfig, EnvironmentConfig +from agent0.base.policies import BasePolicy +from agent0.hyperdrive.exec import run_agents +from chainsync.db.hyperdrive.interface import get_pool_config, get_pool_info, get_transactions, get_wallet_deltas +from chainsync.exec import acquire_data +from eth_account.signers.local import LocalAccount +from ethpy import EthConfig +from ethpy.hyperdrive import HyperdriveAddresses +from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch +from fixedpointmath import FixedPoint +from sqlalchemy.orm import Session + +# This pass is to prevent auto reordering imports from reordering the imports below +pass # pylint: disable=unnecessary-pass + +# Test fixture imports +# Ignoring unused import warning, fixtures are used through variable name +from agent0.test_fixtures import ( # pylint: disable=unused-import, ungrouped-imports + AgentDoneException, + cycle_trade_policy, +) +from chainsync.test_fixtures import db_session # pylint: disable=unused-import, ungrouped-imports +from ethpy.test_fixtures import local_chain, local_hyperdrive_chain # pylint: disable=unused-import, ungrouped-imports # fixture arguments in test function have to be the same as the fixture name # pylint: disable=redefined-outer-name class TestLocalChain: - """CRUD tests for checkpoint table""" + """Tests bringing up local chain""" # This is using 2 fixtures. Since hyperdrive_contract_address depends on local_chain, we need both here # This is due to adding test fixtures through imports - def test_hyperdrive_init_and_deploy(self, local_chain, hyperdrive_contract_address): + def test_hyperdrive_init_and_deploy(self, local_chain: str, local_hyperdrive_chain: dict): """Create and entry""" print(local_chain) - print(hyperdrive_contract_address) + print(local_hyperdrive_chain) + + +def _to_unscaled_decimal(scaled_value: int) -> Decimal: + return Decimal(str(FixedPoint(scaled_value=scaled_value))) + + +def _decimal_almost_equal(a_val: Decimal, b_val: Decimal) -> bool: + return abs(a_val - b_val) < 1e-12 + + +class TestBotToDb: + """Tests pipeline from bots making trades to viewing the trades in the db""" + + # TODO split this up into different functions that work with tests + # pylint: disable=too-many-locals, too-many-statements + def test_bot_to_db( + self, + local_chain: str, + local_hyperdrive_chain: dict, + cycle_trade_policy: Type[BasePolicy], + db_session: Session, + ): + """Runs the entire pipeline and checks the database at the end. + All arguments are fixtures. + """ + # Get hyperdrive chain info + deploy_account: LocalAccount = local_hyperdrive_chain["deploy_account"] + hyperdrive_contract_addresses: HyperdriveAddresses = local_hyperdrive_chain["hyperdrive_contract_addresses"] + + # Build environment config + env_config = EnvironmentConfig( + delete_previous_logs=False, + halt_on_errors=True, + log_filename="system_test", + log_level=logging.INFO, + log_stdout=True, + random_seed=1234, + username="test", + ) + + # Build agent config + agent_config: list[AgentConfig] = [ + AgentConfig( + policy=cycle_trade_policy, + number_of_agents=1, + slippage_tolerance=FixedPoint(0.0001), + base_budget_wei=int(1_000_000e18), # 1 million base + eth_budget_wei=int(100e18), # 100 base + init_kwargs={}, + ), + ] + + # No need for random seed, this bot is deterministic + account_key_config = build_account_key_config_from_agent_config(agent_config) + + # Build custom eth config pointing to local test chain + eth_config = EthConfig( + # Artifacts_url isn't used here, as we explicitly set addresses and passed to run_bots + ARTIFACTS_URL="not_used", + RPC_URL=local_chain, + # Using default abi dir + ) + + # Run bots + try: + run_agents( + env_config, + agent_config, + account_key_config, + develop=True, + eth_config=eth_config, + contract_addresses=hyperdrive_contract_addresses, + ) + except AgentDoneException: + # Using this exception to stop the agents, + # so this exception is expected on test pass + pass + + # Run acquire data to get data from chain to db in subprocess + acquire_data( + start_block=8, # First 7 blocks are deploying hyperdrive, ignore + eth_config=eth_config, + db_session=db_session, + contract_addresses=hyperdrive_contract_addresses, + # Exit the script after catching up to the chain + exit_on_catch_up=True, + ) + + # Test db entries are what we expect + # We don't coerce to float because we want exact values in decimal + db_pool_config_df: pd.DataFrame = get_pool_config(db_session, coerce_float=False) + + # TODO these expected values are defined in lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py + # Eventually, we want to parameterize these values to pass into deploying hyperdrive + expected_timestretch_fp = FixedPoint(scaled_value=_calculateTimeStretch(int(0.05e18))) + # TODO this is actually inv of solidity time stretch, fix + expected_timestretch = _to_unscaled_decimal((1 / expected_timestretch_fp).scaled_value) + expected_inv_timestretch = _to_unscaled_decimal(expected_timestretch_fp.scaled_value) + + expected_pool_config = { + "contractAddress": hyperdrive_contract_addresses.mock_hyperdrive, + "baseToken": hyperdrive_contract_addresses.base_token, + "initialSharePrice": _to_unscaled_decimal(int(1e18)), + "minimumShareReserves": _to_unscaled_decimal(int(10e18)), + "positionDuration": 604800, # 1 week + "checkpointDuration": 3600, # 1 hour + # TODO this is actually inv of solidity time stretch, fix + "timeStretch": expected_timestretch, + "governance": deploy_account.address, + "feeCollector": deploy_account.address, + "curveFee": _to_unscaled_decimal(int(0.1e18)), # 10% + "flatFee": _to_unscaled_decimal(int(0.0005e18)), # 0.05% + "governanceFee": _to_unscaled_decimal(int(0.15e18)), # 15% + "oracleSize": _to_unscaled_decimal(10), + "updateGap": 3600, # TODO don't know where this is getting set + "invTimeStretch": expected_inv_timestretch, + } + + # Existence test + assert len(db_pool_config_df) == 1, "DB must have one entry for pool config" + db_pool_config: pd.Series = db_pool_config_df.iloc[0] + + # Ensure keys match + # Converting to sets and compare + db_keys = set(db_pool_config.index) + expected_keys = set(expected_pool_config.keys()) + assert db_keys == expected_keys, "Keys in db do not match expected" + + # Value comparison + for key, expected_value in expected_pool_config.items(): + # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type + # Internally, they store Numeric types as floats, hence we see rounding errors in testing + # This does not happen in postgres, where these values match exactly. + # https://github.com/delvtech/elf-simulations/issues/836 + + if isinstance(expected_value, Decimal): + assert_val = _decimal_almost_equal(db_pool_config[key], expected_value) + else: + assert_val = db_pool_config[key] == expected_value + + assert assert_val, f"Values do not match for {key} ({db_pool_config[key]} != {expected_value})" + + # Pool info comparison + db_pool_info: pd.DataFrame = get_pool_info(db_session, coerce_float=False) + expected_pool_info_keys = [ + # Keys from contract call + "shareReserves", + "bondReserves", + "lpTotalSupply", + "sharePrice", + "longsOutstanding", + "longAverageMaturityTime", + "shortsOutstanding", + "shortAverageMaturityTime", + "shortBaseVolume", + "withdrawalSharesReadyToWithdraw", + "withdrawalSharesProceeds", + "lpSharePrice", + # Added keys + "timestamp", + # blockNumber is the index of the dataframe + # Calculated keys + "totalSupplyWithdrawalShares", + ] + # Convert to sets and compare + assert set(db_pool_info.columns) == set(expected_pool_info_keys) + + db_transaction_info: pd.DataFrame = get_transactions(db_session, coerce_float=False) + # TODO check transaction keys + # This likely involves cleaning up what columns we grab from transactions + + db_wallet_delta: pd.DataFrame = get_wallet_deltas(db_session, coerce_float=False) + + # Ensure trades exist in database + # Should be 7 total transactions + assert len(db_transaction_info) == 7 + np.testing.assert_array_equal( + db_transaction_info["input_method"], + [ + "addLiquidity", + "openLong", + "openShort", + "removeLiquidity", + "closeLong", + "closeShort", + "redeemWithdrawalShares", + ], + ) + + # 7 total trades in wallet deltas + assert db_wallet_delta["blockNumber"].nunique() == 7 + # 15 different wallet deltas (2 token deltas per trade except for withdraw shares, which is 3) + assert len(db_wallet_delta) == 15 + + actual_num_longs = Decimal("nan") + actual_num_shorts = Decimal("nan") + actual_num_lp = Decimal("nan") + actual_num_withdrawal = Decimal("nan") + # Go through each trade and ensure wallet deltas are correct + # The asserts here are equality because they are either int -> Decimal, which is lossless, + # or they're comparing values after the lossy conversion + for block_number, txn in db_transaction_info.iterrows(): + if txn["input_method"] == "addLiquidity": + assert txn["input_params_contribution"] == Decimal(11111) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + lp_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LP"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(lp_delta_df) == 1 + assert len(base_delta_df) == 1 + lp_delta = lp_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 11111 base for... + assert base_delta["delta"] == -Decimal(11111) + # TODO check LP delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_lp = lp_delta["delta"] + + if txn["input_method"] == "openLong": + assert txn["input_params_baseAmount"] == Decimal(22222) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + long_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LONG"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(long_delta_df) == 1 + assert len(base_delta_df) == 1 + long_delta = long_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 22222 base for... + assert base_delta["delta"] == -Decimal(22222) + # TODO check long delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_longs = long_delta["delta"] + + if txn["input_method"] == "openShort": + assert txn["input_params_bondAmount"] == Decimal(33333) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + short_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "SHORT"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(short_delta_df) == 1 + assert len(base_delta_df) == 1 + short_delta = short_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 33333 bonds for... + assert short_delta["delta"] == Decimal(33333) + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_shorts = short_delta["delta"] + + if txn["input_method"] == "removeLiquidity": + # TODO change this to expected num lp + assert txn["input_params_shares"] == actual_num_lp + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 3 + lp_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LP"] + withdrawal_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "WITHDRAWAL_SHARE"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(lp_delta_df) == 1 + assert len(withdrawal_delta_df) == 1 + assert len(base_delta_df) == 1 + lp_delta = lp_delta_df.iloc[0] + withdrawal_delta = withdrawal_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected lp + assert lp_delta["delta"] == -actual_num_lp + # TODO check base delta + # TODO check withdrawal delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_withdrawal = withdrawal_delta["delta"] + + if txn["input_method"] == "closeLong": + # TODO change this to expected long amount + assert txn["input_params_bondAmount"] == actual_num_longs + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + long_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LONG"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(long_delta_df) == 1 + assert len(base_delta_df) == 1 + long_delta = long_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected longs + assert long_delta["delta"] == -actual_num_longs + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + if txn["input_method"] == "closeShort": + assert txn["input_params_bondAmount"] == Decimal(33333) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + short_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "SHORT"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(short_delta_df) == 1 + assert len(base_delta_df) == 1 + short_delta = short_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected shorts + assert short_delta["delta"] == -actual_num_shorts + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + if txn["input_method"] == "redeemWithdrawalShares": + # TODO change this to expected withdrawal shares + assert txn["input_params_shares"] == actual_num_withdrawal + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + withdrawal_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "WITHDRAWAL_SHARE"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(withdrawal_delta_df) == 1 + assert len(base_delta_df) == 1 + withdrawal_delta = withdrawal_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected withdrawal shares + assert withdrawal_delta["delta"] == -actual_num_withdrawal + # TODO check base delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx