From c85329b40e7cc0fa4cf4fc49b886a17ca78455bf Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 13:08:37 -0700 Subject: [PATCH 01/32] Agent0 docstrings --- lib/agent0/agent0/__init__.py | 7 ++++- lib/agent0/agent0/accounts_config.py | 41 ++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/lib/agent0/agent0/__init__.py b/lib/agent0/agent0/__init__.py index 4dc59c2ffc..d2630e32b4 100644 --- a/lib/agent0/agent0/__init__.py +++ b/lib/agent0/agent0/__init__.py @@ -1,2 +1,7 @@ """Account key config and various helper functions""" -from .accounts_config import AccountKeyConfig, build_account_config_from_env, initialize_accounts +from .accounts_config import ( + AccountKeyConfig, + build_account_config_from_env, + build_account_key_config_from_agent_config, + initialize_accounts, +) diff --git a/lib/agent0/agent0/accounts_config.py b/lib/agent0/agent0/accounts_config.py index 0f98666f8d..6825e11378 100644 --- a/lib/agent0/agent0/accounts_config.py +++ b/lib/agent0/agent0/accounts_config.py @@ -58,13 +58,33 @@ def to_env_str(self) -> str: def initialize_accounts( - agent_config: list[AgentConfig], env_file: str | None = None, random_seed: int = 1, develop: bool = False + agent_config: list[AgentConfig], + env_file: str | None = None, + random_seed: int = 1, + develop: bool = False, ) -> AccountKeyConfig: """ Build or load an accounts environment file. If it doesn't exist, create it based on agent_config. (if develop is off, print instructions on adding in user private key and running script to fund agents). If it does exist, read it in and use it. + + Arguments + --------- + agent_config: list[AgentConfig] + The list of agent configs that define policies and arguments. + env_file: str | None + The path to the env file to write/load from. Defaults to `accounts.env`. + random_seed: int + Random seed to use for initializing budgets. + develop: bool + Flag for development mode. If False, will exit if env_file doesn't exist and print instructions + on how to fund bot. + + Returns + ------- + AccountKeyConfig + The account config object linked to the env file. """ # Default location if env_file is None: @@ -102,21 +122,23 @@ def initialize_accounts( def build_account_key_config_from_agent_config( - agent_configs: list[AgentConfig], random_seed: int, user_key: str | None = None + agent_configs: list[AgentConfig], random_seed: int = 1, user_key: str | None = None ) -> AccountKeyConfig: """Build an Account Config from a provided agent config. Arguments -------- + agent_config: list[AgentConfig] + The list of agent configs that define policies and arguments. + random_seed: int + The seed to initialize the random generator to pass for each bot user_key: str The provided user key to use - agent_configs: list[AgentConfig] - The provided agent configs Returns ------- - AccountConfig - Config settings required to connect to the eth node + AccountKeyConfig + The account config object linked to the env file. """ rng = np.random.default_rng(random_seed) agent_private_keys = [] @@ -151,6 +173,13 @@ def build_account_key_config_from_agent_config( def build_account_config_from_env(env_file: str | None = None, user_key: str | None = None) -> AccountKeyConfig: """Build an Account Config from environmental variables. + Arguments + -------- + env_file: str | None + The path to the env file to load from. Defaults to `accounts.env`. + user_key: str + The provided user key to use + Returns ------- AccountConfig From da6d02a3b125236feaa652cc03fa493681fcfae7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 14:46:50 -0700 Subject: [PATCH 02/32] Moving chainsync main function to be within the package --- lib/chainsync/bin/run_chainsync.py | 24 +++++++++++++++ lib/chainsync/chainsync/exec/__init__.py | 1 + .../{bin => chainsync/exec}/acquire_data.py | 30 +++++-------------- 3 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 lib/chainsync/bin/run_chainsync.py create mode 100644 lib/chainsync/chainsync/exec/__init__.py rename lib/chainsync/{bin => chainsync/exec}/acquire_data.py (88%) diff --git a/lib/chainsync/bin/run_chainsync.py b/lib/chainsync/bin/run_chainsync.py new file mode 100644 index 0000000000..8b72e43b85 --- /dev/null +++ b/lib/chainsync/bin/run_chainsync.py @@ -0,0 +1,24 @@ +"""Script to format on-chain hyperdrive pool, config, and transaction data post-processing.""" +from __future__ import annotations + +from chainsync.exec import acquire_data +from elfpy.utils import logs as log_utils +from ethpy import build_eth_config + +if __name__ == "__main__": + # setup constants + START_BLOCK = 0 + # Look back limit for backfilling + LOOKBACK_BLOCK_LIMIT = 100000 + + # Load parameters from env vars if they exist + config = build_eth_config() + + log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) + acquire_data( + config.ARTIFACTS_URL, + config.RPC_URL, + config.ABI_DIR, + START_BLOCK, + LOOKBACK_BLOCK_LIMIT, + ) diff --git a/lib/chainsync/chainsync/exec/__init__.py b/lib/chainsync/chainsync/exec/__init__.py new file mode 100644 index 0000000000..80fd80d4d7 --- /dev/null +++ b/lib/chainsync/chainsync/exec/__init__.py @@ -0,0 +1 @@ +from .acquire_data import acquire_data diff --git a/lib/chainsync/bin/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py similarity index 88% rename from lib/chainsync/bin/acquire_data.py rename to lib/chainsync/chainsync/exec/acquire_data.py index b766897303..65b50a6a14 100644 --- a/lib/chainsync/bin/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -11,10 +11,8 @@ get_latest_block_number_from_pool_info_table, init_data_chain_to_db, ) -from elfpy.utils import logs as log_utils from eth_typing import URI, BlockNumber from eth_utils import address -from ethpy import build_eth_config from ethpy.base import initialize_web3_with_http_provider, load_all_abis from ethpy.hyperdrive import fetch_hyperdrive_address_from_url from ethpy.hyperdrive.interface import get_hyperdrive_contract @@ -24,12 +22,13 @@ _SLEEP_AMOUNT = 1 -def main( +def acquire_data( artifacts_url: str, rpc_url: URI | str, abi_dir: str, start_block: int, lookback_block_limit: int, + overwrite_session: Session | None = None, ): """Execute the data acquisition pipeline. @@ -48,7 +47,11 @@ def main( """ ## Initialization # postgres session - session = initialize_session() + if overwrite_session is not None: + session = overwrite_session + else: + session = initialize_session() + # web3 provider web3: Web3 = initialize_web3_with_http_provider(rpc_url, request_kwargs={"timeout": 60}) # send a request to the local server to fetch the deployed contract addresses and @@ -105,22 +108,3 @@ def main( continue data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, session) time.sleep(_SLEEP_AMOUNT) - - -if __name__ == "__main__": - # setup constants - START_BLOCK = 0 - # Look back limit for backfilling - LOOKBACK_BLOCK_LIMIT = 100000 - - # Load parameters from env vars if they exist - config = build_eth_config() - - log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) - main( - config.ARTIFACTS_URL, - config.RPC_URL, - config.ABI_DIR, - START_BLOCK, - LOOKBACK_BLOCK_LIMIT, - ) From 37c62a8106c4f840e2d14c6ea680b76c0dbbdd87 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 14:48:13 -0700 Subject: [PATCH 03/32] Chain fixture now returns all addresses. Adding bot to system test --- lib/agent0/agent0/test_fixtures/__init__.py | 2 + .../test_fixtures/cycle_trade_policy.py | 160 ++++++++++++++++++ lib/ethpy/ethpy/test_fixtures/__init__.py | 2 +- lib/ethpy/ethpy/test_fixtures/local_chain.py | 13 +- tests/system_test.py | 96 ++++++++++- 5 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 lib/agent0/agent0/test_fixtures/__init__.py create mode 100644 lib/agent0/agent0/test_fixtures/cycle_trade_policy.py diff --git a/lib/agent0/agent0/test_fixtures/__init__.py b/lib/agent0/agent0/test_fixtures/__init__.py new file mode 100644 index 0000000000..d93fc97880 --- /dev/null +++ b/lib/agent0/agent0/test_fixtures/__init__.py @@ -0,0 +1,2 @@ +"""Test fixtures for agent0""" +from .cycle_trade_policy import AgentDoneException, cycle_trade_policy diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py new file mode 100644 index 0000000000..4581151e40 --- /dev/null +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -0,0 +1,160 @@ +"""Pytest fixture that creates an in memory db session and creates dummy db schemas""" +from __future__ import annotations + +from typing import Type + +import pytest +from agent0.hyperdrive.agents import HyperdriveWallet +from agent0.hyperdrive.policies import HyperdrivePolicy +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState +from elfpy.types import MarketType, Trade +from fixedpointmath import FixedPoint +from numpy.random._generator import Generator as NumpyGenerator + + +class AgentDoneException(Exception): + """Custom exception for signaling the bot is done""" + + pass + + +# Build custom policy +# Simple agent, opens a set of all trades for a fixed amount and closes them after +class CycleTradesPolicy(HyperdrivePolicy): + """A agent that simply cycles through all trades""" + + # Using default parameters + def __init__( + self, + budget: FixedPoint, + rng: NumpyGenerator | None = None, + slippage_tolerance: FixedPoint | None = None, + # Add additional parameters for custom policy here + static_trade_amount_wei: int = int(100e18), # 100 base + ): + self.static_trade_amount_wei = static_trade_amount_wei + # We want to do a sequence of trades one at a time, so we keep an internal counter based on + # how many times `action` has been called. + self.counter = 0 + super().__init__(budget, rng, slippage_tolerance) + + def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMarketAction]]: + """This agent simply opens all trades for a fixed amount and closes them after, one at a time""" + action_list = [] + if self.counter == 0: + # Add liquidity + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.ADD_LIQUIDITY, + trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + wallet=wallet, + ), + ) + ) + elif self.counter == 1: + # Open Long + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_LONG, + trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + wallet=wallet, + ), + ) + ) + elif self.counter == 2: + # Open Short + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_SHORT, + trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + wallet=wallet, + ), + ) + ) + elif self.counter == 3: + # Remove All Liquidity + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.REMOVE_LIQUIDITY, + trade_amount=wallet.lp_tokens, + wallet=wallet, + ), + ) + ) + elif self.counter == 4: + # Close All Longs + assert len(wallet.longs) == 1 + for long_time, long in wallet.longs.items(): + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.CLOSE_LONG, + trade_amount=long.balance, + wallet=wallet, + # TODO is this actually maturity time? Not mint time? + mint_time=long_time, + ), + ) + ) + elif self.counter == 4: + # Close All Shorts + assert len(wallet.shorts) == 1 + for short_time, short in wallet.shorts.items(): + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.CLOSE_SHORT, + trade_amount=short.balance, + wallet=wallet, + # TODO is this actually maturity time? Not mint time? + mint_time=short_time, + ), + ) + ) + elif self.counter == 5: + # Redeem all withdrawal shares + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.REDEEM_WITHDRAW_SHARE, + trade_amount=wallet.withdraw_shares, + wallet=wallet, + ), + ) + ) + elif self.counter == 6: + # One more dummy trade to ensure the previous trades get into the db + # TODO test if we can remove this eventually + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_LONG, + trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + wallet=wallet, + ), + ) + ) + else: + # We want this bot to exit and crash after it's done the trades it needs to do + raise AgentDoneException("Bot done") + self.counter += 1 + return action_list + + +@pytest.fixture(scope="function") +def cycle_trade_policy() -> Type[CycleTradesPolicy]: + """Test fixture to build a policy that cycles through all trades""" + return CycleTradesPolicy diff --git a/lib/ethpy/ethpy/test_fixtures/__init__.py b/lib/ethpy/ethpy/test_fixtures/__init__.py index a09453224f..24a64db853 100644 --- a/lib/ethpy/ethpy/test_fixtures/__init__.py +++ b/lib/ethpy/ethpy/test_fixtures/__init__.py @@ -1,2 +1,2 @@ """Test fixtures for ethpy""" -from .local_chain import hyperdrive_contract_address, local_chain +from .local_chain import hyperdrive_contract_addresses, local_chain diff --git a/lib/ethpy/ethpy/test_fixtures/local_chain.py b/lib/ethpy/ethpy/test_fixtures/local_chain.py index 29c68c91d4..6ab03fee01 100644 --- a/lib/ethpy/ethpy/test_fixtures/local_chain.py +++ b/lib/ethpy/ethpy/test_fixtures/local_chain.py @@ -5,6 +5,8 @@ import pytest from ethpy.base import initialize_web3_with_http_provider +from ethpy.hyperdrive.addresses import HyperdriveAddresses +from web3 import Web3 from .deploy_hyperdrive import deploy_and_initialize_hyperdrive, deploy_hyperdrive_factory, initialize_deploy_account @@ -41,7 +43,7 @@ def local_chain() -> Generator[str, Any, Any]: @pytest.fixture(scope="function") -def hyperdrive_contract_address(local_chain: str) -> str: +def hyperdrive_contract_addresses(local_chain: str) -> HyperdriveAddresses: """Initializes hyperdrive on a local anvil chain for testing. Returns the hyperdrive contract address. @@ -49,4 +51,11 @@ def hyperdrive_contract_address(local_chain: str) -> str: web3 = initialize_web3_with_http_provider(local_chain, reset_provider=False) account = initialize_deploy_account(web3) base_token_contract, factory_contract = deploy_hyperdrive_factory(local_chain, account) - return deploy_and_initialize_hyperdrive(web3, base_token_contract, factory_contract, account) + hyperdrive_addr = deploy_and_initialize_hyperdrive(web3, base_token_contract, factory_contract, account) + + return HyperdriveAddresses( + base_token=Web3.to_checksum_address(base_token_contract.address), + hyperdrive_factory=Web3.to_checksum_address(factory_contract.address), + mock_hyperdrive=Web3.to_checksum_address(hyperdrive_addr), + mock_hyperdrive_math="not_used", + ) diff --git a/tests/system_test.py b/tests/system_test.py index 4490404ce7..eb6513e5f4 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -1,17 +1,105 @@ """System test for end to end testing of elf-simulations""" +import logging +from typing import Type + +from agent0 import build_account_key_config_from_agent_config +from agent0.base.config import AgentConfig, EnvironmentConfig +from agent0.base.policies import BasePolicy +from agent0.hyperdrive.exec import run_agents +from ethpy import EthConfig +from ethpy.hyperdrive.addresses import HyperdriveAddresses +from fixedpointmath import FixedPoint +from sqlalchemy.orm import Session + +# This pass is to prevent auto reordering imports from reordering the imports below +pass # pylint: disable=unnecessary-pass + +# Test fixture imports +# Ignoring unused import warning, fixtures are used through variable name +from agent0.test_fixtures import ( # pylint: disable=unused-import, ungrouped-imports + AgentDoneException, + cycle_trade_policy, +) from chainsync.test_fixtures import db_session # pylint: disable=unused-import -from ethpy.test_fixtures import hyperdrive_contract_address, local_chain # pylint: disable=unused-import +from ethpy.test_fixtures import ( # pylint: disable=unused-import, ungrouped-imports + hyperdrive_contract_addresses, + local_chain, +) # fixture arguments in test function have to be the same as the fixture name # pylint: disable=redefined-outer-name class TestLocalChain: - """CRUD tests for checkpoint table""" + """Tests bringing up local chain""" # This is using 2 fixtures. Since hyperdrive_contract_address depends on local_chain, we need both here # This is due to adding test fixtures through imports - def test_hyperdrive_init_and_deploy(self, local_chain, hyperdrive_contract_address): + def test_hyperdrive_init_and_deploy(self, local_chain: str, hyperdrive_contract_addresses: HyperdriveAddresses): """Create and entry""" print(local_chain) - print(hyperdrive_contract_address) + print(hyperdrive_contract_addresses) + + +class TestBotToDb: + """Tests pipeline from bots making trades to viewing the trades in the db""" + + # This is using 3 fixtures + def test_bot_to_db( + self, + local_chain: str, + hyperdrive_contract_addresses: HyperdriveAddresses, + cycle_trade_policy: Type[BasePolicy], + db_session: Session, + ): + # Build environment config + env_config = EnvironmentConfig( + delete_previous_logs=False, + halt_on_errors=True, + log_filename="system_test", + log_level=logging.INFO, + log_stdout=True, + random_seed=1234, + username="test", + ) + + # Build agent config + agent_config: list[AgentConfig] = [ + AgentConfig( + policy=cycle_trade_policy, + number_of_agents=1, + slippage_tolerance=FixedPoint(0.0001), + base_budget_wei=int(10_000e18), # 10k base + eth_budget_wei=int(10e18), # 10 base + init_kwargs={"static_trade_amount_wei": int(100e18)}, # 100 base static trades + ), + ] + + # No need for random seed, this bot is deterministic + account_key_config = build_account_key_config_from_agent_config(agent_config) + + # Build custom eth config pointing to local chain + eth_config = EthConfig( + # Artifacts_url isn't used here, as we explicitly set addresses and passed to run_bots + RPC_URL=local_chain, + # Default abi dir + ) + + # Run bots + try: + run_agents( + env_config, + agent_config, + account_key_config, + develop=True, + eth_config=eth_config, + override_addresses=hyperdrive_contract_addresses, + ) + except AgentDoneException: + # Using this exception to stop the agents, + # so this exception is expected on test pass + pass + + # Run acquire data to get data from chain to db + + # TODO ensure all trades are in the db From 654284c89bf372a4c4bcda13a059f237998aef7b Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 14:48:41 -0700 Subject: [PATCH 04/32] Adding import --- lib/chainsync/chainsync/exec/acquire_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index 65b50a6a14..a1e67858f8 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -16,6 +16,7 @@ from ethpy.base import initialize_web3_with_http_provider, load_all_abis from ethpy.hyperdrive import fetch_hyperdrive_address_from_url from ethpy.hyperdrive.interface import get_hyperdrive_contract +from sqlalchemy.orm import Session from web3 import Web3 from web3.contract.contract import Contract From 83695597844c476c2e92cdce2b81c7e65b566631 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:06:46 -0700 Subject: [PATCH 05/32] Moving get_web3_and_hyperdrive_contracts from agent0 to ethpy --- lib/agent0/agent0/hyperdrive/exec/__init__.py | 2 +- .../exec/create_and_fund_user_account.py | 6 +-- .../agent0/hyperdrive/exec/fund_agents.py | 2 +- .../agent0/hyperdrive/exec/run_agents.py | 2 +- .../hyperdrive/exec/setup_experiment.py | 42 +-------------- lib/agent0/bin/fund_agents_from_user_key.py | 2 +- lib/ethpy/ethpy/hyperdrive/__init__.py | 1 + .../get_web3_and_hyperdrive_contracts.py | 53 +++++++++++++++++++ lib/ethpy/ethpy/test_fixtures/local_chain.py | 2 +- 9 files changed, 63 insertions(+), 49 deletions(-) create mode 100644 lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py diff --git a/lib/agent0/agent0/hyperdrive/exec/__init__.py b/lib/agent0/agent0/hyperdrive/exec/__init__.py index 3fc0ccda1d..90778a5c76 100644 --- a/lib/agent0/agent0/hyperdrive/exec/__init__.py +++ b/lib/agent0/agent0/hyperdrive/exec/__init__.py @@ -12,5 +12,5 @@ from .fund_agents import fund_agents from .get_agent_accounts import get_agent_accounts from .run_agents import run_agents -from .setup_experiment import get_web3_and_contracts, register_username, setup_experiment +from .setup_experiment import register_username, setup_experiment from .trade_loop import get_wait_for_new_block, trade_if_new_block diff --git a/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py b/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py index 78b72a0f95..702d3dbb60 100644 --- a/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py +++ b/lib/agent0/agent0/hyperdrive/exec/create_and_fund_user_account.py @@ -7,9 +7,7 @@ from eth_account.account import Account from ethpy import EthConfig from ethpy.base import set_anvil_account_balance, smart_contract_transact -from ethpy.hyperdrive.addresses import HyperdriveAddresses - -from .setup_experiment import get_web3_and_contracts +from ethpy.hyperdrive import HyperdriveAddresses, get_web3_and_hyperdrive_contracts def create_and_fund_user_account( @@ -38,7 +36,7 @@ def create_and_fund_user_account( user_private_key = make_private_key(extra_entropy="FAKE USER") # argument value can be any str user_account = HyperdriveAgent(Account().from_key(user_private_key)) - web3, base_token_contract, _ = get_web3_and_contracts(eth_config, contract_addresses) + web3, base_token_contract, _ = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) eth_balance = sum((int(budget) for budget in account_key_config.AGENT_ETH_BUDGETS)) * 2 # double for good measure _ = set_anvil_account_balance(web3, user_account.address, eth_balance) diff --git a/lib/agent0/agent0/hyperdrive/exec/fund_agents.py b/lib/agent0/agent0/hyperdrive/exec/fund_agents.py index abcdf9005b..ba71e3df43 100644 --- a/lib/agent0/agent0/hyperdrive/exec/fund_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/fund_agents.py @@ -15,7 +15,7 @@ smart_contract_read, smart_contract_transact, ) -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses def fund_agents( diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index b4882eccef..ac7a35f820 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -9,7 +9,7 @@ from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config -from ethpy.hyperdrive.addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url +from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_url from .create_and_fund_user_account import create_and_fund_user_account from .fund_agents import fund_agents diff --git a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py index d01b3dbccb..1f6b3dc002 100644 --- a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py +++ b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py @@ -11,8 +11,7 @@ from agent0.hyperdrive.exec.crash_report import setup_hyperdrive_crash_report_logging from elfpy.utils import logs from ethpy import EthConfig -from ethpy.base import initialize_web3_with_http_provider, load_all_abis -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses, get_web3_and_hyperdrive_contracts from web3 import Web3 from web3.contract.contract import Contract @@ -65,7 +64,7 @@ def setup_experiment( log_format_string=environment_config.log_formatter, ) setup_hyperdrive_crash_report_logging() - web3, base_token_contract, hyperdrive_contract = get_web3_and_contracts(eth_config, contract_addresses) + web3, base_token_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) # load agent policies # rng is shared by the agents and can be accessed via `agent_accounts[idx].policy.rng` agent_accounts = get_agent_accounts( @@ -74,43 +73,6 @@ def setup_experiment( return web3, base_token_contract, hyperdrive_contract, agent_accounts -def get_web3_and_contracts( - eth_config: EthConfig, contract_addresses: HyperdriveAddresses -) -> tuple[Web3, Contract, Contract]: - """Get the web3 container and the ERC20Base and Hyperdrive contracts. - - Arguments - --------- - eth_config: EthConfig - Configuration for urls to the rpc and artifacts. - contract_addresses: HyperdriveAddresses - Configuration for defining various contract addresses. - - Returns - ------- - tuple[Web3, Contract, Contract] - A tuple containing: - - The web3 container - - The base token contract - - The hyperdrive contract - """ - # point to chain env - web3 = initialize_web3_with_http_provider(eth_config.RPC_URL, reset_provider=False) - # setup base contract interface - abis = load_all_abis(eth_config.ABI_DIR) - # set up the ERC20 contract for minting base tokens - # TODO is there a better way to pass in base and hyperdrive abi? - base_token_contract: Contract = web3.eth.contract( - abi=abis["ERC20Mintable"], address=web3.to_checksum_address(contract_addresses.base_token) - ) - # set up hyperdrive contract - hyperdrive_contract: Contract = web3.eth.contract( - abi=abis["IHyperdrive"], - address=web3.to_checksum_address(contract_addresses.mock_hyperdrive), - ) - return web3, base_token_contract, hyperdrive_contract - - def register_username(register_url: str, wallet_addrs: list[str], username: str) -> None: """Registers the username with the flask server. diff --git a/lib/agent0/bin/fund_agents_from_user_key.py b/lib/agent0/bin/fund_agents_from_user_key.py index 02b238b1d9..f86bf187ae 100644 --- a/lib/agent0/bin/fund_agents_from_user_key.py +++ b/lib/agent0/bin/fund_agents_from_user_key.py @@ -9,7 +9,7 @@ from agent0.hyperdrive.exec import fund_agents from eth_account.account import Account from ethpy import build_eth_config -from ethpy.hyperdrive.addresses import fetch_hyperdrive_address_from_url +from ethpy.hyperdrive import fetch_hyperdrive_address_from_url if __name__ == "__main__": parser = argparse.ArgumentParser( diff --git a/lib/ethpy/ethpy/hyperdrive/__init__.py b/lib/ethpy/ethpy/hyperdrive/__init__.py index 9053dbf18b..47256d3e00 100644 --- a/lib/ethpy/ethpy/hyperdrive/__init__.py +++ b/lib/ethpy/ethpy/hyperdrive/__init__.py @@ -2,6 +2,7 @@ from .addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url from .assets import AssetIdPrefix, decode_asset_id, encode_asset_id from .errors import HyperdriveErrors, lookup_hyperdrive_error_selector +from .get_web3_and_hyperdrive_contracts import get_web3_and_hyperdrive_contracts from .interface import ( get_hyperdrive_checkpoint_info, get_hyperdrive_config, diff --git a/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py b/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py new file mode 100644 index 0000000000..5036650e21 --- /dev/null +++ b/lib/ethpy/ethpy/hyperdrive/get_web3_and_hyperdrive_contracts.py @@ -0,0 +1,53 @@ +"""Helper function for getting web3 and contracts.""" +from __future__ import annotations + +import os + +from ethpy import EthConfig +from ethpy.base import initialize_web3_with_http_provider, load_all_abis +from web3 import Web3 +from web3.contract.contract import Contract + +from .addresses import HyperdriveAddresses, fetch_hyperdrive_address_from_url + + +def get_web3_and_hyperdrive_contracts( + eth_config: EthConfig, contract_addresses: HyperdriveAddresses | None = None +) -> tuple[Web3, Contract, Contract]: + """Get the web3 container and the ERC20Base and Hyperdrive contracts. + + Arguments + --------- + eth_config: EthConfig + Configuration for urls to the rpc and artifacts. + contract_addresses: HyperdriveAddresses | None + Configuration for defining various contract addresses. + Will query eth_config artifacts for addresses by default + + Returns + ------- + tuple[Web3, Contract, Contract] + A tuple containing: + - The web3 container + - The base token contract + - The hyperdrive contract + """ + # Initialize contract addresses if none + if contract_addresses is None: + contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) + + # point to chain env + web3 = initialize_web3_with_http_provider(eth_config.RPC_URL, reset_provider=False) + # setup base contract interface + abis = load_all_abis(eth_config.ABI_DIR) + # set up the ERC20 contract for minting base tokens + # TODO is there a better way to pass in base and hyperdrive abi? + base_token_contract: Contract = web3.eth.contract( + abi=abis["ERC20Mintable"], address=web3.to_checksum_address(contract_addresses.base_token) + ) + # set up hyperdrive contract + hyperdrive_contract: Contract = web3.eth.contract( + abi=abis["IHyperdrive"], + address=web3.to_checksum_address(contract_addresses.mock_hyperdrive), + ) + return web3, base_token_contract, hyperdrive_contract diff --git a/lib/ethpy/ethpy/test_fixtures/local_chain.py b/lib/ethpy/ethpy/test_fixtures/local_chain.py index 6ab03fee01..bce4fd1438 100644 --- a/lib/ethpy/ethpy/test_fixtures/local_chain.py +++ b/lib/ethpy/ethpy/test_fixtures/local_chain.py @@ -5,7 +5,7 @@ import pytest from ethpy.base import initialize_web3_with_http_provider -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses from web3 import Web3 from .deploy_hyperdrive import deploy_and_initialize_hyperdrive, deploy_hyperdrive_factory, initialize_deploy_account From cc39c9a34e0a5fe71815c6acc81a499065e60ca1 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:09:54 -0700 Subject: [PATCH 06/32] Using new ethpy utilities to get web3 and contracts in chainsync --- lib/chainsync/bin/run_chainsync.py | 7 --- lib/chainsync/chainsync/exec/acquire_data.py | 45 +++++++++----------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/lib/chainsync/bin/run_chainsync.py b/lib/chainsync/bin/run_chainsync.py index 8b72e43b85..c08b30f4ff 100644 --- a/lib/chainsync/bin/run_chainsync.py +++ b/lib/chainsync/bin/run_chainsync.py @@ -3,7 +3,6 @@ from chainsync.exec import acquire_data from elfpy.utils import logs as log_utils -from ethpy import build_eth_config if __name__ == "__main__": # setup constants @@ -11,14 +10,8 @@ # Look back limit for backfilling LOOKBACK_BLOCK_LIMIT = 100000 - # Load parameters from env vars if they exist - config = build_eth_config() - log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) acquire_data( - config.ARTIFACTS_URL, - config.RPC_URL, - config.ABI_DIR, START_BLOCK, LOOKBACK_BLOCK_LIMIT, ) diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index a1e67858f8..1b1dd21585 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -11,10 +11,11 @@ get_latest_block_number_from_pool_info_table, init_data_chain_to_db, ) -from eth_typing import URI, BlockNumber +from eth_typing import BlockNumber from eth_utils import address +from ethpy import EthConfig, build_eth_config from ethpy.base import initialize_web3_with_http_provider, load_all_abis -from ethpy.hyperdrive import fetch_hyperdrive_address_from_url +from ethpy.hyperdrive import fetch_hyperdrive_address_from_url, get_web3_and_hyperdrive_contracts from ethpy.hyperdrive.interface import get_hyperdrive_contract from sqlalchemy.orm import Session from web3 import Web3 @@ -24,46 +25,40 @@ def acquire_data( - artifacts_url: str, - rpc_url: URI | str, - abi_dir: str, start_block: int, lookback_block_limit: int, - overwrite_session: Session | None = None, + eth_config: EthConfig | None, + overwrite_db_session: Session | None = None, ): """Execute the data acquisition pipeline. Arguments --------- - artifacts_url: str - The url of the artifacts server from which we get addresses. - rpc_url: URI | str - The url to the ethereum node - abi_dir : str - The path to the abi directory start_block : int The starting block to filter the query on lookback_block_limit : int The maximum number of blocks to look back when filling in missing data + eth_config: EthConfig | None + Configuration for urls to the rpc and artifacts. If not set, will look for addresses + in eth.env. + overwrite_db_session: Session | None + Session object for connecting to db. If None, will initialize a new session based on + postgres.env. """ ## Initialization + # eth config + if eth_config is None: + # Load parameters from env vars if they exist + eth_config = build_eth_config() + # postgres session - if overwrite_session is not None: - session = overwrite_session + if overwrite_db_session is not None: + session = overwrite_db_session else: session = initialize_session() - # web3 provider - web3: Web3 = initialize_web3_with_http_provider(rpc_url, request_kwargs={"timeout": 60}) - # send a request to the local server to fetch the deployed contract addresses and - # all Hyperdrive contract addresses from the server response - addresses = fetch_hyperdrive_address_from_url(os.path.join(artifacts_url, "addresses.json")) - abis = load_all_abis(abi_dir) - # Contracts - hyperdrive_contract = get_hyperdrive_contract(web3, abis, addresses) - base_contract: Contract = web3.eth.contract( - address=address.to_checksum_address(addresses.base_token), abi=abis["ERC20Mintable"] - ) + # Get web3 and contracts + web3, base_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config) ## Get starting point for restarts # Get last entry of pool info in db From 37a46df5e6da5bb6bd04acba5eeb71898eb1c47c Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:10:44 -0700 Subject: [PATCH 07/32] Cleanup of acquire_data --- lib/chainsync/chainsync/exec/acquire_data.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index 1b1dd21585..ace0a2731c 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -12,14 +12,9 @@ init_data_chain_to_db, ) from eth_typing import BlockNumber -from eth_utils import address from ethpy import EthConfig, build_eth_config -from ethpy.base import initialize_web3_with_http_provider, load_all_abis -from ethpy.hyperdrive import fetch_hyperdrive_address_from_url, get_web3_and_hyperdrive_contracts -from ethpy.hyperdrive.interface import get_hyperdrive_contract +from ethpy.hyperdrive import get_web3_and_hyperdrive_contracts from sqlalchemy.orm import Session -from web3 import Web3 -from web3.contract.contract import Contract _SLEEP_AMOUNT = 1 @@ -27,7 +22,7 @@ def acquire_data( start_block: int, lookback_block_limit: int, - eth_config: EthConfig | None, + eth_config: EthConfig | None = None, overwrite_db_session: Session | None = None, ): """Execute the data acquisition pipeline. From a0399df8135b3cc7cb92b014d8a50ab612969c3d Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:11:02 -0700 Subject: [PATCH 08/32] More cleanup --- lib/chainsync/chainsync/exec/acquire_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index ace0a2731c..c7eeee35cd 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -2,7 +2,6 @@ from __future__ import annotations import logging -import os import time from chainsync.db.base import initialize_session From 4e17a190b213a17da13b703f705b42912029c53e Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:11:28 -0700 Subject: [PATCH 09/32] Minor import change --- tests/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system_test.py b/tests/system_test.py index eb6513e5f4..a2bb00c009 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -7,7 +7,7 @@ from agent0.base.policies import BasePolicy from agent0.hyperdrive.exec import run_agents from ethpy import EthConfig -from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.hyperdrive import HyperdriveAddresses from fixedpointmath import FixedPoint from sqlalchemy.orm import Session From e65d650cf742006f1f8fcadda17bc3278f12c503 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:35:43 -0700 Subject: [PATCH 10/32] Parameterizing acquire data for system tests --- .../agent0/hyperdrive/exec/run_agents.py | 18 ++++----- lib/chainsync/bin/run_chainsync.py | 10 +---- lib/chainsync/chainsync/exec/acquire_data.py | 37 ++++++++++++------- 3 files changed, 33 insertions(+), 32 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index ac7a35f820..157ea4e315 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -26,7 +26,7 @@ def run_agents( account_key_config: AccountKeyConfig, develop: bool = False, eth_config: EthConfig | None = None, - override_addresses: HyperdriveAddresses | None = None, + contract_addresses: HyperdriveAddresses | None = None, ) -> None: """Entrypoint to run agents. @@ -43,24 +43,22 @@ def run_agents( eth_config: EthConfig | None Configuration for urls to the rpc and artifacts. If not set, will look for addresses in eth.env. - override_addresses: HyperdriveAddresses | None + contract_addresses: HyperdriveAddresses | None If set, will use these addresses instead of querying the artifact url defined in eth_config. """ - # Defaults to looking for eth_config env - if eth_config is None: - eth_config = build_eth_config() - # Set sane logging defaults to avoid spam from dependencies logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("web3").setLevel(logging.WARNING) warnings.filterwarnings("ignore", category=UserWarning, module="web3.contract.base_contract") - # Get addresses either from artifacts url defined in eth_config or from override_addresses - if override_addresses is not None: - contract_addresses = override_addresses - else: + # Defaults to looking for eth_config env + if eth_config is None: + eth_config = build_eth_config() + + # Get addresses either from artifacts url defined in eth_config or from contract_addresses + if contract_addresses is None: contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) if develop: # setup env automatically & fund the agents diff --git a/lib/chainsync/bin/run_chainsync.py b/lib/chainsync/bin/run_chainsync.py index c08b30f4ff..b004936ce1 100644 --- a/lib/chainsync/bin/run_chainsync.py +++ b/lib/chainsync/bin/run_chainsync.py @@ -5,13 +5,5 @@ from elfpy.utils import logs as log_utils if __name__ == "__main__": - # setup constants - START_BLOCK = 0 - # Look back limit for backfilling - LOOKBACK_BLOCK_LIMIT = 100000 - log_utils.setup_logging(".logging/acquire_data.log", log_stdout=True) - acquire_data( - START_BLOCK, - LOOKBACK_BLOCK_LIMIT, - ) + acquire_data() diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index c7eeee35cd..643efdae65 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import os import time from chainsync.db.base import initialize_session @@ -12,17 +13,21 @@ ) from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config -from ethpy.hyperdrive import get_web3_and_hyperdrive_contracts +from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_url, get_web3_and_hyperdrive_contracts from sqlalchemy.orm import Session _SLEEP_AMOUNT = 1 +# Lots of arguments +# pylint: disable=too-many-arguments def acquire_data( - start_block: int, - lookback_block_limit: int, + start_block: int = 0, + lookback_block_limit: int = 10000, eth_config: EthConfig | None = None, - overwrite_db_session: Session | None = None, + db_session: Session | None = None, + contract_addresses: HyperdriveAddresses | None = None, + exit_on_catch_up: bool = False, ): """Execute the data acquisition pipeline. @@ -38,6 +43,8 @@ def acquire_data( overwrite_db_session: Session | None Session object for connecting to db. If None, will initialize a new session based on postgres.env. + exit_on_catch_up: bool + If True, will exit after catching up to current block """ ## Initialization # eth config @@ -46,17 +53,19 @@ def acquire_data( eth_config = build_eth_config() # postgres session - if overwrite_db_session is not None: - session = overwrite_db_session - else: - session = initialize_session() + if db_session is None: + db_session = initialize_session() + + # Get addresses either from artifacts url defined in eth_config or from contract_addresses + if contract_addresses is None: + contract_addresses = fetch_hyperdrive_address_from_url(os.path.join(eth_config.ARTIFACTS_URL, "addresses.json")) # Get web3 and contracts - web3, base_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config) + web3, base_contract, hyperdrive_contract = get_web3_and_hyperdrive_contracts(eth_config, contract_addresses) ## Get starting point for restarts # Get last entry of pool info in db - data_latest_block_number = get_latest_block_number_from_pool_info_table(session) + data_latest_block_number = get_latest_block_number_from_pool_info_table(db_session) # Using max of latest block in database or specified start block block_number: BlockNumber = BlockNumber(max(start_block, data_latest_block_number)) # Make sure to not grab current block, as the current block is subject to change @@ -69,11 +78,11 @@ def acquire_data( logging.warning("Starting block is past lookback block limit, starting at block %s", block_number) # Collect initial data - init_data_chain_to_db(hyperdrive_contract, session) + init_data_chain_to_db(hyperdrive_contract, db_session) # This if statement executes only on initial run (based on data_latest_block_number check), # and if the chain has executed until start_block (based on latest_mined_block check) if data_latest_block_number < block_number < latest_mined_block: - data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, session) + data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, db_session) # Main data loop # monitor for new blocks & add pool info per block @@ -83,6 +92,8 @@ def acquire_data( # Only execute if we are on a new block if latest_mined_block <= block_number: time.sleep(_SLEEP_AMOUNT) + if exit_on_catch_up: + break continue # Backfilling for blocks that need updating for block_int in range(block_number + 1, latest_mined_block + 1): @@ -96,5 +107,5 @@ def acquire_data( latest_mined_block, ) continue - data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, session) + data_chain_to_db(web3, base_contract, hyperdrive_contract, block_number, db_session) time.sleep(_SLEEP_AMOUNT) From 31779c59f3816125bba58e56358e536eccb1c81e Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 15:36:07 -0700 Subject: [PATCH 11/32] Updating system tests to include acquire_data --- tests/system_test.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/system_test.py b/tests/system_test.py index a2bb00c009..a40f6143d7 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -6,6 +6,7 @@ from agent0.base.config import AgentConfig, EnvironmentConfig from agent0.base.policies import BasePolicy from agent0.hyperdrive.exec import run_agents +from chainsync.exec import acquire_data from ethpy import EthConfig from ethpy.hyperdrive import HyperdriveAddresses from fixedpointmath import FixedPoint @@ -81,8 +82,9 @@ def test_bot_to_db( # Build custom eth config pointing to local chain eth_config = EthConfig( # Artifacts_url isn't used here, as we explicitly set addresses and passed to run_bots + ARTIFACTS_URL="not_used", RPC_URL=local_chain, - # Default abi dir + # Using default abi dir ) # Run bots @@ -93,13 +95,20 @@ def test_bot_to_db( account_key_config, develop=True, eth_config=eth_config, - override_addresses=hyperdrive_contract_addresses, + contract_addresses=hyperdrive_contract_addresses, ) except AgentDoneException: # Using this exception to stop the agents, # so this exception is expected on test pass pass - # Run acquire data to get data from chain to db + # Run acquire data to get data from chain to db in subprocess + acquire_data( + eth_config=eth_config, + db_session=db_session, + contract_addresses=hyperdrive_contract_addresses, + # Exit the script after catching up to the chain + exit_on_catch_up=True, + ) # TODO ensure all trades are in the db From 6cf16cf4f6b682b9f23510c0e661d6695f0669fa Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:33:47 -0700 Subject: [PATCH 12/32] Adding in conftest to allow for vscode debugging of pytest --- conftest.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000000..5a06825bb0 --- /dev/null +++ b/conftest.py @@ -0,0 +1,33 @@ +# Hack to allow for vscode debugger to throw exception immediately +# instead of allowing pytest to catch the exception and report +# Use this in conjunction with the following launch.json configuration: +# { +# "name": "Debug Tests", +# "type": "python", +# "request": "launch", +# "module": "pytest", +# "args": ["${file}"], +# "console": "integratedTerminal", +# "justMyCode": true, +# "env": { +# "_PYTEST_RAISE": "1" +# }, +# }, + +# Ignore docstrings for this file +# pylint: disable=missing-docstring + + +import os + +import pytest + +if os.getenv("_PYTEST_RAISE", "0") != "0": + + @pytest.hookimpl(tryfirst=True) + def pytest_exception_interact(call): + raise call.excinfo.value + + @pytest.hookimpl(tryfirst=True) + def pytest_internalerror(excinfo): + raise excinfo.value From f59e6421496a7fb1414af2147b8d220f8cfef7d0 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:35:43 -0700 Subject: [PATCH 13/32] Linking to source --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index 5a06825bb0..dd7cf53af2 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,7 @@ # Hack to allow for vscode debugger to throw exception immediately # instead of allowing pytest to catch the exception and report +# Based on https://stackoverflow.com/questions/62419998/how-can-i-get-pytest-to-not-catch-exceptions/62563106#62563106 + # Use this in conjunction with the following launch.json configuration: # { # "name": "Debug Tests", From 3b1eced40a6b58f0ae766d217ed334ba24075a9f Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:53:16 -0700 Subject: [PATCH 14/32] Updating debug conf test name --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index dd7cf53af2..8abc9633f2 100644 --- a/conftest.py +++ b/conftest.py @@ -4,7 +4,7 @@ # Use this in conjunction with the following launch.json configuration: # { -# "name": "Debug Tests", +# "name": "Debug Current Test", # "type": "python", # "request": "launch", # "module": "pytest", From 6b90a6f7892c291e4b248af20fead60f1a31c44e Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:54:12 -0700 Subject: [PATCH 15/32] Fixing bug with inconsistent db schema between BigInteger and Decimal for maturity time --- lib/chainsync/chainsync/db/hyperdrive/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index 148463b338..15460cb75f 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -91,7 +91,7 @@ class WalletInfo(Base): # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) tokenValue: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), default=None) + maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) @@ -115,7 +115,7 @@ class WalletDelta(Base): # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) delta: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - maturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) class HyperdriveTransaction(Base): @@ -168,7 +168,7 @@ class HyperdriveTransaction(Base): # input_params_asUnderlying # Method: closeLong - input_params_maturityTime: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + input_params_maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) # input_params_bondAmount # input_params_minOutput # input_params_destination @@ -206,7 +206,7 @@ class HyperdriveTransaction(Base): event_id: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) # Fields calculated from base event_prefix: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - event_maturity_time: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + event_maturity_time: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) # Fields not used by postprocessing From 0cf8c5c5e36cecf901f14691b88997d6924de7a4 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:54:25 -0700 Subject: [PATCH 16/32] Adjusting starting block for system test --- tests/system_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system_test.py b/tests/system_test.py index a40f6143d7..53efad902f 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -104,6 +104,7 @@ def test_bot_to_db( # Run acquire data to get data from chain to db in subprocess acquire_data( + start_block=8, # First 7 blocks are deploying hyperdrive, ignore eth_config=eth_config, db_session=db_session, contract_addresses=hyperdrive_contract_addresses, From be16171bf59fbc42f8417f1cbae2df8c0bd5f5a3 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 16:54:54 -0700 Subject: [PATCH 17/32] Removing unnecessary pass --- lib/agent0/agent0/test_fixtures/cycle_trade_policy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py index 4581151e40..58a22d27e1 100644 --- a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -16,8 +16,6 @@ class AgentDoneException(Exception): """Custom exception for signaling the bot is done""" - pass - # Build custom policy # Simple agent, opens a set of all trades for a fixed amount and closes them after From a310dc6b681f27875974fba494fefa4ec4bbe3f5 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 18:08:30 -0700 Subject: [PATCH 18/32] Resolving timestretch todo --- lib/chainsync/bin/run_hyperdrive_dashboard.py | 3 --- lib/ethpy/ethpy/hyperdrive/interface.py | 5 ++--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/chainsync/bin/run_hyperdrive_dashboard.py b/lib/chainsync/bin/run_hyperdrive_dashboard.py index f49e39af59..93d9ab16c6 100644 --- a/lib/chainsync/bin/run_hyperdrive_dashboard.py +++ b/lib/chainsync/bin/run_hyperdrive_dashboard.py @@ -34,9 +34,6 @@ # pool config data is static, so just read once config_data = get_pool_config(session, coerce_float=False) -# TODO fix input invTimeStretch to be unscaled in ingestion into postgres -config_data["invTimeStretch"] = config_data["invTimeStretch"] / 10**18 - config_data = config_data.iloc[0] diff --git a/lib/ethpy/ethpy/hyperdrive/interface.py b/lib/ethpy/ethpy/hyperdrive/interface.py index d573b4dac0..7b218fcf6e 100644 --- a/lib/ethpy/ethpy/hyperdrive/interface.py +++ b/lib/ethpy/ethpy/hyperdrive/interface.py @@ -116,9 +116,8 @@ def get_hyperdrive_config(hyperdrive_contract: Contract) -> dict[str, Any]: # Ok so, the contracts store the time stretch constant in an inverted manner from the python. # In order to not break the world, we save the contract version as 'invTimeStretch' and invert # that to get the python version 'timeStretch' - # TODO invTimeStretch should be in fixed point notation - pool_config["invTimeStretch"] = hyperdrive_config["timeStretch"] - pool_config["timeStretch"] = FixedPoint(1) / FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) + pool_config["invTimeStretch"] = FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) + pool_config["timeStretch"] = FixedPoint(1) / pool_config["invTimeStretch"] pool_config["governance"] = hyperdrive_config["governance"] pool_config["feeCollector"] = hyperdrive_config["feeCollector"] curve_fee, flat_fee, governance_fee = hyperdrive_config["fees"] From 0ab14669729ba5e443a53932098e8273677cf5aa Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 18:50:49 -0700 Subject: [PATCH 19/32] Using explicit scale in postgres to match fixedpoint notation --- .../chainsync/db/hyperdrive/convert_data.py | 2 +- .../chainsync/db/hyperdrive/schema.py | 85 ++++++++++--------- 2 files changed, 46 insertions(+), 41 deletions(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/convert_data.py b/lib/chainsync/chainsync/db/hyperdrive/convert_data.py index 30c43db1f6..caa19effc2 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/convert_data.py +++ b/lib/chainsync/chainsync/db/hyperdrive/convert_data.py @@ -565,7 +565,7 @@ def _build_hyperdrive_transaction_object( "transactionHash": transaction_dict["hash"], "txn_to": transaction_dict["to"], "txn_from": transaction_dict["from"], - "gasUsed": receipt["gasUsed"], + "gasUsed": _convert_scaled_value_to_decimal(receipt["gasUsed"]), } # Input solidity methods and parameters # TODO can the input field ever be empty or not exist? diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index 15460cb75f..4c071ef375 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -10,6 +10,9 @@ # pylint: disable=invalid-name +# Postgres numeric type that matches fixedpoint +FIXED_NUMERIC = Numeric(precision=1000, scale=18) + class PoolConfig(Base): """Table/dataclass schema for pool config.""" @@ -18,20 +21,20 @@ class PoolConfig(Base): contractAddress: Mapped[str] = mapped_column(String, primary_key=True) baseToken: Mapped[Union[str, None]] = mapped_column(String, default=None) - initialSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - minimumShareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + initialSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + minimumShareReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) positionDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None) checkpointDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - timeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + timeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) governance: Mapped[Union[str, None]] = mapped_column(String, default=None) feeCollector: Mapped[Union[str, None]] = mapped_column(String, default=None) - curveFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - flatFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - governanceFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - oracleSize: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + curveFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + flatFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + governanceFee: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + oracleSize: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None) - invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - termLength: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + termLength: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) class CheckpointInfo(Base): @@ -41,9 +44,9 @@ class CheckpointInfo(Base): blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True) timestamp: Mapped[datetime] = mapped_column(DateTime, index=True) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) class PoolInfo(Base): @@ -56,19 +59,19 @@ class PoolInfo(Base): blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True) timestamp: Mapped[datetime] = mapped_column(DateTime, index=True) - shareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - bondReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - lpTotalSupply: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - lpSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - longAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - withdrawalSharesReadyToWithdraw: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - withdrawalSharesProceeds: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - totalSupplyWithdrawalShares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + shareReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + bondReserves: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + lpTotalSupply: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + lpSharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + longAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + withdrawalSharesReadyToWithdraw: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + withdrawalSharesProceeds: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + totalSupplyWithdrawalShares: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # TODO: Rename this to something more accurate to what is happening, e.g. HyperdriveTransactions @@ -90,9 +93,9 @@ class WalletInfo(Base): baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None) # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) - tokenValue: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + tokenValue: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) - sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + sharePrice: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # TODO: either make a more general TokenDelta, or rename this to HyperdriveDelta @@ -114,7 +117,7 @@ class WalletDelta(Base): baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None) # tokenType is the baseTokenType appended with "-" for LONG and SHORT tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None) - delta: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + delta: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) @@ -142,7 +145,7 @@ class HyperdriveTransaction(Base): # Almost always from wallet address to smart contract address txn_to: Mapped[Union[str, None]] = mapped_column(String, default=None) txn_from: Mapped[Union[str, None]] = mapped_column(String, default=None) - gasUsed: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + gasUsed: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) #### Fields from solidity function calls #### # These fields map solidity function calls and their corresponding arguments @@ -150,20 +153,20 @@ class HyperdriveTransaction(Base): input_method: Mapped[Union[str, None]] = mapped_column(String, default=None) # Method: initialize - input_params_contribution: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_apr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_contribution: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_apr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) input_params_destination: Mapped[Union[str, None]] = mapped_column(String, default=None) input_params_asUnderlying: Mapped[Union[bool, None]] = mapped_column(Boolean, default=None) # Method: openLong - input_params_baseAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_minOutput: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_baseAmount: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_minOutput: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying # Method: openShort - input_params_bondAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_maxDeposit: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_bondAmount: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_maxDeposit: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying @@ -183,13 +186,13 @@ class HyperdriveTransaction(Base): # Method: addLiquidity # input_params_contribution - input_params_minApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) - input_params_maxApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_minApr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + input_params_maxApr: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_destination # input_params_asUnderlying # Method: removeLiquidity - input_params_shares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + input_params_shares: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) # input_params_minOutput # input_params_destination # input_params_asUnderlying @@ -201,9 +204,11 @@ class HyperdriveTransaction(Base): # args_owner # args_spender # args_id - event_value: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None) + event_value: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) event_operator: Mapped[Union[str, None]] = mapped_column(String, default=None) - event_id: Mapped[Union[int, None]] = mapped_column(Numeric, default=None) + event_id: Mapped[Union[int, None]] = mapped_column( + Numeric, default=None + ) # Integer too small here to store event_id, so we use Numeric here instead # Fields calculated from base event_prefix: Mapped[Union[int, None]] = mapped_column(Integer, default=None) event_maturity_time: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) From 3b0094f1e8d7d03aa6cb9f81b230647082fd4fd0 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 18:51:14 -0700 Subject: [PATCH 20/32] First steps in comparing values in system test --- tests/system_test.py | 49 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/tests/system_test.py b/tests/system_test.py index 53efad902f..30f0f34db6 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -1,14 +1,18 @@ """System test for end to end testing of elf-simulations""" import logging +from decimal import Decimal from typing import Type +import pandas as pd from agent0 import build_account_key_config_from_agent_config from agent0.base.config import AgentConfig, EnvironmentConfig from agent0.base.policies import BasePolicy from agent0.hyperdrive.exec import run_agents +from chainsync.db.hyperdrive.interface import get_pool_config from chainsync.exec import acquire_data from ethpy import EthConfig from ethpy.hyperdrive import HyperdriveAddresses +from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch from fixedpointmath import FixedPoint from sqlalchemy.orm import Session @@ -42,6 +46,10 @@ def test_hyperdrive_init_and_deploy(self, local_chain: str, hyperdrive_contract_ print(hyperdrive_contract_addresses) +def _to_unscaled_decimal(scaled_value: int) -> Decimal: + return Decimal(str(FixedPoint(scaled_value=scaled_value))) + + class TestBotToDb: """Tests pipeline from bots making trades to viewing the trades in the db""" @@ -112,4 +120,43 @@ def test_bot_to_db( exit_on_catch_up=True, ) - # TODO ensure all trades are in the db + # Run acquire data to get data from chain to db in subprocess + acquire_data( + start_block=8, # First 7 blocks are deploying hyperdrive, ignore + eth_config=eth_config, + db_session=db_session, + contract_addresses=hyperdrive_contract_addresses, + # Exit the script after catching up to the chain + exit_on_catch_up=True, + ) + + # Test db entries are what we expect + # We don't coerce to float because we want exact values in decimal + db_pool_config = get_pool_config(db_session, coerce_float=False) + + # TODO these expected values are defined in lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py + # Eventually, we want to parameterize these values to pass into deploying hyperdrive + expected_timestretch_fp = FixedPoint(scaled_value=_calculateTimeStretch(int(0.05e18))) + # TODO this is actually inv of solidity time stretch, fix + expected_timestretch = _to_unscaled_decimal((1 / expected_timestretch_fp).scaled_value) + expected_inv_timestretch = _to_unscaled_decimal(expected_timestretch_fp.scaled_value) + + expected_pool_config = pd.Series( + { + "contractAddress": hyperdrive_contract_addresses.mock_hyperdrive, + "baseToken": hyperdrive_contract_addresses.base_token, + "initialSharePrice": _to_unscaled_decimal(int(1e18)), + "minimumShareReserves": _to_unscaled_decimal(int(10e18)), + "positionDuration": 604800, # 1 week + "checkpointDuration": 3600, # 1 hour + # TODO this is actually inv of solidity time stretch, fix + "timeStretch": expected_timestretch, + # "governance": + # "feeCollector": + "invTimeStretch": expected_timestretch, + } + ) + + # TODO timestretch has rounding error + + # Ensure all trades are in the db From aaf710e86f325658d89fcc8e8e8ec9f0244f1dcd Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 18:59:16 -0700 Subject: [PATCH 21/32] Adding comments on high precision numeric types in postgres --- lib/chainsync/chainsync/db/hyperdrive/schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index 4c071ef375..44450d0519 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -11,6 +11,8 @@ # pylint: disable=invalid-name # Postgres numeric type that matches fixedpoint +# The high precision doesn't actually allocate memory in postgres, as numeric is variable size +# https://stackoverflow.com/questions/40686571/performance-of-numeric-type-with-high-precisions-and-scales-in-postgresql FIXED_NUMERIC = Numeric(precision=1000, scale=18) From a65a5e19dc8d74c006284bdbe8c95e1a1b12ffe8 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 19:04:05 -0700 Subject: [PATCH 22/32] Updating comments and docstrings --- lib/chainsync/chainsync/db/hyperdrive/schema.py | 2 +- lib/chainsync/chainsync/exec/acquire_data.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index 44450d0519..bd4fa4c0d6 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -210,7 +210,7 @@ class HyperdriveTransaction(Base): event_operator: Mapped[Union[str, None]] = mapped_column(String, default=None) event_id: Mapped[Union[int, None]] = mapped_column( Numeric, default=None - ) # Integer too small here to store event_id, so we use Numeric here instead + ) # Integer too small here to store event_id, so we use Numeric instead # Fields calculated from base event_prefix: Mapped[Union[int, None]] = mapped_column(Integer, default=None) event_maturity_time: Mapped[Union[int, None]] = mapped_column(BigInteger, default=None) diff --git a/lib/chainsync/chainsync/exec/acquire_data.py b/lib/chainsync/chainsync/exec/acquire_data.py index 643efdae65..a338ae7621 100644 --- a/lib/chainsync/chainsync/exec/acquire_data.py +++ b/lib/chainsync/chainsync/exec/acquire_data.py @@ -40,9 +40,12 @@ def acquire_data( eth_config: EthConfig | None Configuration for urls to the rpc and artifacts. If not set, will look for addresses in eth.env. - overwrite_db_session: Session | None + db_session: Session | None Session object for connecting to db. If None, will initialize a new session based on postgres.env. + contract_addresses: HyperdriveAddresses | None + If set, will use these addresses instead of querying the artifact url + defined in eth_config. exit_on_catch_up: bool If True, will exit after catching up to current block """ From ea69d86b0c6afb4c2c7eb70f8fa6fbd5ef481740 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Wed, 16 Aug 2023 19:06:47 -0700 Subject: [PATCH 23/32] Removing extra acquire_data in system test --- tests/system_test.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/system_test.py b/tests/system_test.py index 30f0f34db6..f3b62440d0 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -120,16 +120,6 @@ def test_bot_to_db( exit_on_catch_up=True, ) - # Run acquire data to get data from chain to db in subprocess - acquire_data( - start_block=8, # First 7 blocks are deploying hyperdrive, ignore - eth_config=eth_config, - db_session=db_session, - contract_addresses=hyperdrive_contract_addresses, - # Exit the script after catching up to the chain - exit_on_catch_up=True, - ) - # Test db entries are what we expect # We don't coerce to float because we want exact values in decimal db_pool_config = get_pool_config(db_session, coerce_float=False) From cb2ba50fd780bf2cc663dd0b54286c8813ad085e Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 09:44:21 -0700 Subject: [PATCH 24/32] Fixing tests and adding pool config check to system test --- .../chainsync/db/hyperdrive/interface_test.py | 30 ++++--- .../chainsync/db/hyperdrive/schema.py | 2 +- lib/ethpy/ethpy/hyperdrive/interface.py | 1 + lib/ethpy/ethpy/test_fixtures/__init__.py | 2 +- .../ethpy/test_fixtures/deploy_hyperdrive.py | 4 +- lib/ethpy/ethpy/test_fixtures/local_chain.py | 44 +++++++--- tests/system_test.py | 81 +++++++++++++------ 7 files changed, 112 insertions(+), 52 deletions(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py index ea2464261d..ced7a00a77 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py @@ -149,43 +149,49 @@ def test_get_pool_config(self, db_session): pool_config_1 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config_1, db_session) - pool_config_df_1 = get_pool_config(db_session, coerce_float=False) + pool_config_df_1 = get_pool_config(db_session) assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type + # Internally, they store Numeric types as floats, hence we see rounding errors in testing + # This does not happen in postgres, where these values match exactly. + # https://github.com/delvtech/elf-simulations/issues/836 + np.testing.assert_array_equal(pool_config_df_1["initialSharePrice"], np.array([3.2])) pool_config_2 = PoolConfig(contractAddress="1", initialSharePrice=Decimal("3.4")) add_pool_config(pool_config_2, db_session) - pool_config_df_2 = get_pool_config(db_session, coerce_float=False) + pool_config_df_2 = get_pool_config(db_session) assert len(pool_config_df_2) == 2 - np.testing.assert_array_equal(pool_config_df_2["initialSharePrice"], np.array([Decimal("3.2"), Decimal("3.4")])) + np.testing.assert_array_equal(pool_config_df_2["initialSharePrice"], np.array([3.2, 3.4])) def test_primary_id_query_pool_config(self, db_session): """Testing retrieval of pool config via interface""" pool_config = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config, db_session) - pool_config_df_1 = get_pool_config(db_session, contract_address="0", coerce_float=False) + pool_config_df_1 = get_pool_config(db_session, contract_address="0") assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_1.loc[0, "initialSharePrice"] == 3.2 - pool_config_df_2 = get_pool_config(db_session, contract_address="1", coerce_float=False) + pool_config_df_2 = get_pool_config(db_session, contract_address="1") assert len(pool_config_df_2) == 0 def test_pool_config_verify(self, db_session): """Testing retrieval of pool config via interface""" pool_config_1 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) add_pool_config(pool_config_1, db_session) - pool_config_df_1 = get_pool_config(db_session, coerce_float=False) + pool_config_df_1 = get_pool_config(db_session) assert len(pool_config_df_1) == 1 - assert pool_config_df_1.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_1.loc[0, "initialSharePrice"] == 3.2 # Nothing should happen if we give the same pool_config - pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.2")) + # TODO Below is a hack due to sqlite not having numerics + # We explicitly print 18 spots after floating point to match rounding error in sqlite + pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("{:.18f}".format(3.2))) add_pool_config(pool_config_2, db_session) - pool_config_df_2 = get_pool_config(db_session, coerce_float=False) + pool_config_df_2 = get_pool_config(db_session) assert len(pool_config_df_2) == 1 - assert pool_config_df_2.loc[0, "initialSharePrice"] == Decimal("3.2") + assert pool_config_df_2.loc[0, "initialSharePrice"] == 3.2 # If we try to add another pool config with a different value, should throw a ValueError pool_config_3 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("3.4")) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index bd4fa4c0d6..bf0f1684ee 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -36,7 +36,7 @@ class PoolConfig(Base): oracleSize: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None) invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) - termLength: Mapped[Union[Decimal, None]] = mapped_column(FIXED_NUMERIC, default=None) + updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None) class CheckpointInfo(Base): diff --git a/lib/ethpy/ethpy/hyperdrive/interface.py b/lib/ethpy/ethpy/hyperdrive/interface.py index 7b218fcf6e..f677175af6 100644 --- a/lib/ethpy/ethpy/hyperdrive/interface.py +++ b/lib/ethpy/ethpy/hyperdrive/interface.py @@ -116,6 +116,7 @@ def get_hyperdrive_config(hyperdrive_contract: Contract) -> dict[str, Any]: # Ok so, the contracts store the time stretch constant in an inverted manner from the python. # In order to not break the world, we save the contract version as 'invTimeStretch' and invert # that to get the python version 'timeStretch' + # TODO fix this pool_config["invTimeStretch"] = FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) pool_config["timeStretch"] = FixedPoint(1) / pool_config["invTimeStretch"] pool_config["governance"] = hyperdrive_config["governance"] diff --git a/lib/ethpy/ethpy/test_fixtures/__init__.py b/lib/ethpy/ethpy/test_fixtures/__init__.py index 24a64db853..3465d2a585 100644 --- a/lib/ethpy/ethpy/test_fixtures/__init__.py +++ b/lib/ethpy/ethpy/test_fixtures/__init__.py @@ -1,2 +1,2 @@ """Test fixtures for ethpy""" -from .local_chain import hyperdrive_contract_addresses, local_chain +from .local_chain import local_chain, local_hyperdrive_chain diff --git a/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py b/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py index 7a3cbd9d80..571302dc6a 100644 --- a/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py +++ b/lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py @@ -84,10 +84,10 @@ def deploy_hyperdrive_factory(rpc_url: str, deploy_account: LocalAccount) -> tup initial_variable_rate = int(0.05e18) curve_fee = int(0.1e18) # 10% flat_fee = int(0.0005e18) # 0.05% - governance_fee = int(0.15e18) # 0.15% + governance_fee = int(0.15e18) # 15% max_curve_fee = int(0.3e18) # 30% max_flat_fee = int(0.0015e18) # 0.15% - max_governance_fee = int(0.30e18) # 0.30% + max_governance_fee = int(0.30e18) # 30% # Configuration settings abi_folder = "packages/hyperdrive/src/abis/" diff --git a/lib/ethpy/ethpy/test_fixtures/local_chain.py b/lib/ethpy/ethpy/test_fixtures/local_chain.py index bce4fd1438..510ea8fbe6 100644 --- a/lib/ethpy/ethpy/test_fixtures/local_chain.py +++ b/lib/ethpy/ethpy/test_fixtures/local_chain.py @@ -16,8 +16,12 @@ @pytest.fixture(scope="function") def local_chain() -> Generator[str, Any, Any]: - """Launches a local anvil chain for testing. - Returns the chain url. + """Launches a local anvil chain for testing. Kills the anvil chain after. + + Returns + ------- + Generator[str, Any, Any] + Yields the local anvil chain url """ anvil_port = 9999 host = "127.0.0.1" # localhost @@ -33,7 +37,7 @@ def local_chain() -> Generator[str, Any, Any]: local_chain_ = "http://" + host + ":" + str(anvil_port) - # Hack, wait for anvil chain to initialize + # TODO Hack, wait for anvil chain to initialize time.sleep(3) yield local_chain_ @@ -43,19 +47,39 @@ def local_chain() -> Generator[str, Any, Any]: @pytest.fixture(scope="function") -def hyperdrive_contract_addresses(local_chain: str) -> HyperdriveAddresses: +def local_hyperdrive_chain(local_chain: str) -> dict: """Initializes hyperdrive on a local anvil chain for testing. Returns the hyperdrive contract address. + Arguments + --------- + local_chain: str + The `local_chain` test fixture that binds to the local anvil chain rpc url + + Returns + ------- + dict + A dictionary with the following key - value fields: + + "web3": Web3 + web3 provider object + "deploy_account": LocalAccount + The local account that deploys and initializes hyperdrive + "hyperdrive_contract_addresses": HyperdriveAddresses + The hyperdrive contract addresses """ web3 = initialize_web3_with_http_provider(local_chain, reset_provider=False) account = initialize_deploy_account(web3) base_token_contract, factory_contract = deploy_hyperdrive_factory(local_chain, account) hyperdrive_addr = deploy_and_initialize_hyperdrive(web3, base_token_contract, factory_contract, account) - return HyperdriveAddresses( - base_token=Web3.to_checksum_address(base_token_contract.address), - hyperdrive_factory=Web3.to_checksum_address(factory_contract.address), - mock_hyperdrive=Web3.to_checksum_address(hyperdrive_addr), - mock_hyperdrive_math="not_used", - ) + return { + "web3": web3, + "deploy_account": account, + "hyperdrive_contract_addresses": HyperdriveAddresses( + base_token=Web3.to_checksum_address(base_token_contract.address), + hyperdrive_factory=Web3.to_checksum_address(factory_contract.address), + mock_hyperdrive=Web3.to_checksum_address(hyperdrive_addr), + mock_hyperdrive_math="not_used", + ), + } diff --git a/tests/system_test.py b/tests/system_test.py index f3b62440d0..16a5325473 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -10,11 +10,13 @@ from agent0.hyperdrive.exec import run_agents from chainsync.db.hyperdrive.interface import get_pool_config from chainsync.exec import acquire_data +from eth_account.signers.local import LocalAccount from ethpy import EthConfig from ethpy.hyperdrive import HyperdriveAddresses from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch from fixedpointmath import FixedPoint from sqlalchemy.orm import Session +from web3 import Web3 # This pass is to prevent auto reordering imports from reordering the imports below pass # pylint: disable=unnecessary-pass @@ -26,10 +28,7 @@ cycle_trade_policy, ) from chainsync.test_fixtures import db_session # pylint: disable=unused-import -from ethpy.test_fixtures import ( # pylint: disable=unused-import, ungrouped-imports - hyperdrive_contract_addresses, - local_chain, -) +from ethpy.test_fixtures import local_chain, local_hyperdrive_chain # pylint: disable=unused-import, ungrouped-imports # fixture arguments in test function have to be the same as the fixture name # pylint: disable=redefined-outer-name @@ -40,10 +39,10 @@ class TestLocalChain: # This is using 2 fixtures. Since hyperdrive_contract_address depends on local_chain, we need both here # This is due to adding test fixtures through imports - def test_hyperdrive_init_and_deploy(self, local_chain: str, hyperdrive_contract_addresses: HyperdriveAddresses): + def test_hyperdrive_init_and_deploy(self, local_chain: str, local_hyperdrive_chain: dict): """Create and entry""" print(local_chain) - print(hyperdrive_contract_addresses) + print(local_hyperdrive_chain) def _to_unscaled_decimal(scaled_value: int) -> Decimal: @@ -53,14 +52,19 @@ def _to_unscaled_decimal(scaled_value: int) -> Decimal: class TestBotToDb: """Tests pipeline from bots making trades to viewing the trades in the db""" - # This is using 3 fixtures + # TODO split this up into different functions that work with tests def test_bot_to_db( self, local_chain: str, - hyperdrive_contract_addresses: HyperdriveAddresses, + local_hyperdrive_chain: dict, cycle_trade_policy: Type[BasePolicy], db_session: Session, ): + # Get hyperdrive chain info + web3: Web3 = local_hyperdrive_chain["web3"] + deploy_account: LocalAccount = local_hyperdrive_chain["deploy_account"] + hyperdrive_contract_addresses: HyperdriveAddresses = local_hyperdrive_chain["hyperdrive_contract_addresses"] + # Build environment config env_config = EnvironmentConfig( delete_previous_logs=False, @@ -122,7 +126,7 @@ def test_bot_to_db( # Test db entries are what we expect # We don't coerce to float because we want exact values in decimal - db_pool_config = get_pool_config(db_session, coerce_float=False) + db_pool_config_df: pd.DataFrame = get_pool_config(db_session, coerce_float=False) # TODO these expected values are defined in lib/ethpy/ethpy/test_fixtures/deploy_hyperdrive.py # Eventually, we want to parameterize these values to pass into deploying hyperdrive @@ -131,22 +135,47 @@ def test_bot_to_db( expected_timestretch = _to_unscaled_decimal((1 / expected_timestretch_fp).scaled_value) expected_inv_timestretch = _to_unscaled_decimal(expected_timestretch_fp.scaled_value) - expected_pool_config = pd.Series( - { - "contractAddress": hyperdrive_contract_addresses.mock_hyperdrive, - "baseToken": hyperdrive_contract_addresses.base_token, - "initialSharePrice": _to_unscaled_decimal(int(1e18)), - "minimumShareReserves": _to_unscaled_decimal(int(10e18)), - "positionDuration": 604800, # 1 week - "checkpointDuration": 3600, # 1 hour - # TODO this is actually inv of solidity time stretch, fix - "timeStretch": expected_timestretch, - # "governance": - # "feeCollector": - "invTimeStretch": expected_timestretch, - } - ) - - # TODO timestretch has rounding error + expected_values = { + "contractAddress": hyperdrive_contract_addresses.mock_hyperdrive, + "baseToken": hyperdrive_contract_addresses.base_token, + "initialSharePrice": _to_unscaled_decimal(int(1e18)), + "minimumShareReserves": _to_unscaled_decimal(int(10e18)), + "positionDuration": 604800, # 1 week + "checkpointDuration": 3600, # 1 hour + # TODO this is actually inv of solidity time stretch, fix + "timeStretch": expected_timestretch, + "governance": deploy_account.address, + "feeCollector": deploy_account.address, + "curveFee": _to_unscaled_decimal(int(0.1e18)), # 10% + "flatFee": _to_unscaled_decimal(int(0.0005e18)), # 0.05% + "governanceFee": _to_unscaled_decimal(int(0.15e18)), # 15% + "oracleSize": _to_unscaled_decimal(10), + "updateGap": 3600, # TODO don't know where this is getting set + "invTimeStretch": expected_inv_timestretch, + } + + # Existence test + assert len(db_pool_config_df) == 1, "DB must have one entry for pool config" + db_pool_config: pd.Series = db_pool_config_df.iloc[0] + + # Ensure keys match + # Converting to sets and compare + db_keys = set(db_pool_config.index) + expected_keys = set(expected_values.keys()) + assert db_keys == expected_keys, "Keys in db do not match expected" + + # Value comparison + for key, expected_value in expected_values.items(): + # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type + # Internally, they store Numeric types as floats, hence we see rounding errors in testing + # This does not happen in postgres, where these values match exactly. + # https://github.com/delvtech/elf-simulations/issues/836 + + if isinstance(expected_value, Decimal): + assert_val = abs(db_pool_config[key] - expected_value) < 1e-12 + else: + assert_val = db_pool_config[key] == expected_value + + assert assert_val, f"Values do not match for {key} ({db_pool_config[key]} != {expected_value})" # Ensure all trades are in the db From 8e40c3e13b0c9e83494906602a3bf94bae52dbbc Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 10:27:02 -0700 Subject: [PATCH 25/32] Fixing a bug in cycle_trade_policy --- lib/agent0/agent0/test_fixtures/cycle_trade_policy.py | 6 +++--- lib/agent0/examples/example_agent.py | 9 ++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py index 58a22d27e1..0f5c443842 100644 --- a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -104,7 +104,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 4: + elif self.counter == 5: # Close All Shorts assert len(wallet.shorts) == 1 for short_time, short in wallet.shorts.items(): @@ -120,7 +120,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 5: + elif self.counter == 6: # Redeem all withdrawal shares action_list.append( Trade( @@ -132,7 +132,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 6: + elif self.counter == 7: # One more dummy trade to ensure the previous trades get into the db # TODO test if we can remove this eventually action_list.append( diff --git a/lib/agent0/examples/example_agent.py b/lib/agent0/examples/example_agent.py index 8a51d8df00..9987b561ee 100644 --- a/lib/agent0/examples/example_agent.py +++ b/lib/agent0/examples/example_agent.py @@ -24,6 +24,9 @@ # Build custom policy # Simple agent, opens a set of all trades for a fixed amount and closes them after +# TODO this bot is almost identical to the one defined in test_fixtures for system tests +# On one hand, this bot is nice for an example since it shows all trades +# On the other, duplicated code between the two bots class CycleTradesPolicy(HyperdrivePolicy): """An agent that simply cycles through all trades""" @@ -109,7 +112,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 4: + elif self.counter == 5: # Close All Shorts assert len(wallet.shorts) == 1 for short_time, short in wallet.shorts.items(): @@ -125,7 +128,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 5: + elif self.counter == 6: # Redeem all withdrawal shares action_list.append( Trade( @@ -137,7 +140,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ), ) ) - elif self.counter == 6: + elif self.counter == 7: # One more dummy trade to ensure the previous trades get into the db # TODO test if we can remove this eventually action_list.append( From 0befebe8d7237767d094123883115930412f3e49 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 10:27:22 -0700 Subject: [PATCH 26/32] Adding coerce float to other getter functions. Updating system tests --- .../chainsync/db/hyperdrive/interface.py | 64 +++++++++++++----- tests/system_test.py | 65 +++++++++++++++++-- 2 files changed, 108 insertions(+), 21 deletions(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface.py b/lib/chainsync/chainsync/db/hyperdrive/interface.py index 358ee74b70..c64b124790 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface.py @@ -205,6 +205,8 @@ def get_pool_info( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -230,7 +232,9 @@ def get_pool_info( return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_transactions(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_transactions( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> pd.DataFrame: """Get all transactions and returns as a pandas dataframe. Arguments @@ -243,6 +247,8 @@ def get_transactions(session: Session, start_block: int | None = None, end_block end_block : int | None The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -262,10 +268,12 @@ def get_transactions(session: Session, start_block: int | None = None, end_block if end_block is not None: query = query.filter(HyperdriveTransaction.blockNumber < end_block) - return pd.read_sql(query.statement, con=session.connection()).set_index("blockNumber") + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_checkpoint_info(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_checkpoint_info( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> pd.DataFrame: """Get all info associated with a given checkpoint. This includes @@ -283,6 +291,8 @@ def get_checkpoint_info(session: Session, start_block: int | None = None, end_bl end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -305,10 +315,12 @@ def get_checkpoint_info(session: Session, start_block: int | None = None, end_bl # Always sort by time in order query = query.order_by(CheckpointInfo.timestamp) - return pd.read_sql(query.statement, con=session.connection()).set_index("blockNumber") + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float).set_index("blockNumber") -def get_all_wallet_info(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame: +def get_all_wallet_info( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float: bool = True +) -> pd.DataFrame: """Get all of the wallet_info data in history and returns as a pandas dataframe. Arguments @@ -321,6 +333,8 @@ def get_all_wallet_info(session: Session, start_block: int | None = None, end_bl end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -340,16 +354,18 @@ def get_all_wallet_info(session: Session, start_block: int | None = None, end_bl if end_block is not None: query = query.filter(WalletInfo.blockNumber < end_block) - return pd.read_sql(query.statement, con=session.connection()) + return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) -def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: +def get_wallet_info_history(session: Session, coerce_float=True) -> dict[str, pd.DataFrame]: """Get the history of all wallet info over block time. Arguments --------- session : Session The initialized session object + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -359,8 +375,8 @@ def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: token the address has at that block number, plus a timestamp and the share price of the block """ # Get data - all_wallet_info = get_all_wallet_info(session) - pool_info_lookup = get_pool_info(session)[["timestamp", "sharePrice"]] + all_wallet_info = get_all_wallet_info(session, coerce_float=coerce_float) + pool_info_lookup = get_pool_info(session, coerce_float=coerce_float)[["timestamp", "sharePrice"]] # Pivot tokenType to columns, keeping walletAddress and blockNumber all_wallet_info = all_wallet_info.pivot( @@ -389,7 +405,7 @@ def get_wallet_info_history(session: Session) -> dict[str, pd.DataFrame]: def get_current_wallet_info( - session: Session, start_block: int | None = None, end_block: int | None = None + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float: bool = True ) -> pd.DataFrame: """Get the balance of a wallet and a given end_block. @@ -409,13 +425,17 @@ def get_current_wallet_info( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- DataFrame A DataFrame that consists of the queried wallet info data """ - all_wallet_info = get_all_wallet_info(session, start_block=start_block, end_block=end_block) + all_wallet_info = get_all_wallet_info( + session, start_block=start_block, end_block=end_block, coerce_float=coerce_float + ) # Get last entry in the table of each wallet address and token type # This should always return a dataframe # Pandas doesn't play nice with types @@ -458,6 +478,8 @@ def get_wallet_deltas( end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -480,7 +502,9 @@ def get_wallet_deltas( return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) -def get_all_traders(session: Session, start_block: int | None = None, end_block: int | None = None) -> list[str]: +def get_all_traders( + session: Session, start_block: int | None = None, end_block: int | None = None, coerce_float=True +) -> list[str]: """Get the list of all traders from the WalletInfo table. Arguments @@ -493,6 +517,8 @@ def get_all_traders(session: Session, start_block: int | None = None, end_block: end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -515,12 +541,14 @@ def get_all_traders(session: Session, start_block: int | None = None, end_block: return [] query = query.distinct() - results = pd.read_sql(query.statement, con=session.connection()) + results = pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float) return results["walletAddress"].to_list() -def get_agent_positions(session: Session, filter_addr: list[str] | None = None) -> dict[str, AgentPosition]: +def get_agent_positions( + session: Session, filter_addr: list[str] | None = None, coerce_float: bool = True +) -> dict[str, AgentPosition]: """Create an AgentPosition for each agent in the wallet history. Arguments @@ -529,6 +557,8 @@ def get_agent_positions(session: Session, filter_addr: list[str] | None = None) The initialized session object filter_addr : list[str] | None Only return these addresses. Returns all if None + coerce_float : bool + If true, will return floats in dataframe. Otherwise, will return fixed point Decimal Returns ------- @@ -536,9 +566,11 @@ def get_agent_positions(session: Session, filter_addr: list[str] | None = None) Returns a dictionary keyed by wallet address, value of an agent's position """ if filter_addr is None: - return {agent: AgentPosition(wallet) for agent, wallet in get_wallet_info_history(session).items()} + return { + agent: AgentPosition(wallet) for agent, wallet in get_wallet_info_history(session, coerce_float).items() + } return { agent: AgentPosition(wallet) - for agent, wallet in get_wallet_info_history(session).items() + for agent, wallet in get_wallet_info_history(session, coerce_float).items() if agent in filter_addr } diff --git a/tests/system_test.py b/tests/system_test.py index 16a5325473..2da948f33b 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -3,12 +3,13 @@ from decimal import Decimal from typing import Type +import numpy as np import pandas as pd from agent0 import build_account_key_config_from_agent_config from agent0.base.config import AgentConfig, EnvironmentConfig from agent0.base.policies import BasePolicy from agent0.hyperdrive.exec import run_agents -from chainsync.db.hyperdrive.interface import get_pool_config +from chainsync.db.hyperdrive.interface import get_pool_config, get_pool_info, get_transactions, get_wallet_deltas from chainsync.exec import acquire_data from eth_account.signers.local import LocalAccount from ethpy import EthConfig @@ -91,7 +92,7 @@ def test_bot_to_db( # No need for random seed, this bot is deterministic account_key_config = build_account_key_config_from_agent_config(agent_config) - # Build custom eth config pointing to local chain + # Build custom eth config pointing to local test chain eth_config = EthConfig( # Artifacts_url isn't used here, as we explicitly set addresses and passed to run_bots ARTIFACTS_URL="not_used", @@ -135,7 +136,7 @@ def test_bot_to_db( expected_timestretch = _to_unscaled_decimal((1 / expected_timestretch_fp).scaled_value) expected_inv_timestretch = _to_unscaled_decimal(expected_timestretch_fp.scaled_value) - expected_values = { + expected_pool_config = { "contractAddress": hyperdrive_contract_addresses.mock_hyperdrive, "baseToken": hyperdrive_contract_addresses.base_token, "initialSharePrice": _to_unscaled_decimal(int(1e18)), @@ -161,11 +162,11 @@ def test_bot_to_db( # Ensure keys match # Converting to sets and compare db_keys = set(db_pool_config.index) - expected_keys = set(expected_values.keys()) + expected_keys = set(expected_pool_config.keys()) assert db_keys == expected_keys, "Keys in db do not match expected" # Value comparison - for key, expected_value in expected_values.items(): + for key, expected_value in expected_pool_config.items(): # TODO In testing, we use sqlite, which does not implement the fixed point Numeric type # Internally, they store Numeric types as floats, hence we see rounding errors in testing # This does not happen in postgres, where these values match exactly. @@ -178,4 +179,58 @@ def test_bot_to_db( assert assert_val, f"Values do not match for {key} ({db_pool_config[key]} != {expected_value})" + # Pool info comparison + db_pool_info: pd.DataFrame = get_pool_info(db_session, coerce_float=False) + expected_pool_info_keys = [ + # Keys from contract call + "shareReserves", + "bondReserves", + "lpTotalSupply", + "sharePrice", + "longsOutstanding", + "longAverageMaturityTime", + "shortsOutstanding", + "shortAverageMaturityTime", + "shortBaseVolume", + "withdrawalSharesReadyToWithdraw", + "withdrawalSharesProceeds", + "lpSharePrice", + # Added keys + "timestamp", + # blockNumber is the index of the dataframe + # Calculated keys + "totalSupplyWithdrawalShares", + ] + # Convert to sets and compare + assert set(db_pool_info.columns) == set(expected_pool_info_keys) + + db_transaction_info: pd.DataFrame = get_transactions(db_session, coerce_float=False) + db_wallet_delta: pd.DataFrame = get_wallet_deltas(db_session, coerce_float=False) + + # Ensure trades exist in database + # Should be 7 total transactions + assert len(db_transaction_info) == 7 + np.testing.assert_array_equal( + db_transaction_info["input_method"], + [ + "addLiquidity", + "openLong", + "openShort", + "removeLiquidity", + "closeLong", + "closeShort", + "redeemWithdrawalShares", + ], + ) + + # 7 total trades in wallet deltas + assert db_wallet_delta["blockNumber"].nunique() == 7 + # 15 different wallet deltas (2 token deltas per trade except for withdraw shares, which is 3) + assert len(db_wallet_delta) == 15 + + # TODO + expected_pool_info = {} + + pass + # Ensure all trades are in the db From ea0b63fb0e4cbdc996bae24f5d0fc48ef246f9c5 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 11:49:20 -0700 Subject: [PATCH 27/32] System test now goes through txns and checks wallets --- .../test_fixtures/cycle_trade_policy.py | 14 +- tests/system_test.py | 159 ++++++++++++++++-- 2 files changed, 155 insertions(+), 18 deletions(-) diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py index 0f5c443842..2efa71893e 100644 --- a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -28,10 +28,7 @@ def __init__( budget: FixedPoint, rng: NumpyGenerator | None = None, slippage_tolerance: FixedPoint | None = None, - # Add additional parameters for custom policy here - static_trade_amount_wei: int = int(100e18), # 100 base ): - self.static_trade_amount_wei = static_trade_amount_wei # We want to do a sequence of trades one at a time, so we keep an internal counter based on # how many times `action` has been called. self.counter = 0 @@ -47,7 +44,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( action_type=HyperdriveActionType.ADD_LIQUIDITY, - trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + trade_amount=FixedPoint(scaled_value=int(11111e18)), wallet=wallet, ), ) @@ -59,7 +56,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( action_type=HyperdriveActionType.OPEN_LONG, - trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + trade_amount=FixedPoint(scaled_value=int(22222e18)), wallet=wallet, ), ) @@ -71,7 +68,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( action_type=HyperdriveActionType.OPEN_SHORT, - trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + trade_amount=FixedPoint(scaled_value=int(33333e18)), wallet=wallet, ), ) @@ -134,13 +131,14 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis ) elif self.counter == 7: # One more dummy trade to ensure the previous trades get into the db - # TODO test if we can remove this eventually + # TODO test if we can remove this eventually by allowing acquire_data to look at + # current block action_list.append( Trade( market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( action_type=HyperdriveActionType.OPEN_LONG, - trade_amount=FixedPoint(scaled_value=self.static_trade_amount_wei), + trade_amount=FixedPoint(scaled_value=int(1e18)), wallet=wallet, ), ) diff --git a/tests/system_test.py b/tests/system_test.py index 2da948f33b..90e491cda9 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -50,6 +50,10 @@ def _to_unscaled_decimal(scaled_value: int) -> Decimal: return Decimal(str(FixedPoint(scaled_value=scaled_value))) +def _decimal_almost_equal(a: Decimal, b: Decimal) -> bool: + return abs(a - b) < 1e-12 + + class TestBotToDb: """Tests pipeline from bots making trades to viewing the trades in the db""" @@ -83,9 +87,9 @@ def test_bot_to_db( policy=cycle_trade_policy, number_of_agents=1, slippage_tolerance=FixedPoint(0.0001), - base_budget_wei=int(10_000e18), # 10k base - eth_budget_wei=int(10e18), # 10 base - init_kwargs={"static_trade_amount_wei": int(100e18)}, # 100 base static trades + base_budget_wei=int(1_000_000e18), # 1 million base + eth_budget_wei=int(100e18), # 100 base + init_kwargs={}, ), ] @@ -173,7 +177,7 @@ def test_bot_to_db( # https://github.com/delvtech/elf-simulations/issues/836 if isinstance(expected_value, Decimal): - assert_val = abs(db_pool_config[key] - expected_value) < 1e-12 + assert_val = _decimal_almost_equal(db_pool_config[key], expected_value) else: assert_val = db_pool_config[key] == expected_value @@ -205,6 +209,9 @@ def test_bot_to_db( assert set(db_pool_info.columns) == set(expected_pool_info_keys) db_transaction_info: pd.DataFrame = get_transactions(db_session, coerce_float=False) + # TODO check transaction keys + # This likely involves cleaning up what columns we grab from transactions + db_wallet_delta: pd.DataFrame = get_wallet_deltas(db_session, coerce_float=False) # Ensure trades exist in database @@ -228,9 +235,141 @@ def test_bot_to_db( # 15 different wallet deltas (2 token deltas per trade except for withdraw shares, which is 3) assert len(db_wallet_delta) == 15 - # TODO - expected_pool_info = {} - - pass - - # Ensure all trades are in the db + actual_num_longs = Decimal("nan") + actual_num_shorts = Decimal("nan") + actual_num_lp = Decimal("nan") + actual_num_withdrawal = Decimal("nan") + # Go through each trade and ensure wallet deltas are correct + # The asserts here are equality because they are either int -> Decimal, which is lossless, + # or they're comparing values after the lossy conversion + for block_number, txn in db_transaction_info.iterrows(): + if txn["input_method"] == "addLiquidity": + assert txn["input_params_contribution"] == Decimal(11111) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + lp_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LP"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(lp_delta_df) == 1 + assert len(base_delta_df) == 1 + lp_delta = lp_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 11111 base for... + assert base_delta["delta"] == -Decimal(11111) + # TODO check LP delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_lp = lp_delta["delta"] + + if txn["input_method"] == "openLong": + assert txn["input_params_baseAmount"] == Decimal(22222) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + long_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LONG"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(long_delta_df) == 1 + assert len(base_delta_df) == 1 + long_delta = long_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 22222 base for... + assert base_delta["delta"] == -Decimal(22222) + # TODO check long delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_longs = long_delta["delta"] + + if txn["input_method"] == "openShort": + assert txn["input_params_bondAmount"] == Decimal(33333) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + short_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "SHORT"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(short_delta_df) == 1 + assert len(base_delta_df) == 1 + short_delta = short_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # 33333 bonds for... + assert short_delta["delta"] == Decimal(33333) + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_shorts = short_delta["delta"] + + if txn["input_method"] == "removeLiquidity": + # TODO change this to expected num lp + assert txn["input_params_shares"] == actual_num_lp + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 3 + lp_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LP"] + withdrawal_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "WITHDRAWAL_SHARE"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(lp_delta_df) == 1 + assert len(withdrawal_delta_df) == 1 + assert len(base_delta_df) == 1 + lp_delta = lp_delta_df.iloc[0] + withdrawal_delta = withdrawal_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected lp + assert lp_delta["delta"] == -actual_num_lp + # TODO check base delta + # TODO check withdrawal delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + actual_num_withdrawal = withdrawal_delta["delta"] + + if txn["input_method"] == "closeLong": + # TODO change this to expected long amount + assert txn["input_params_bondAmount"] == actual_num_longs + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + long_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "LONG"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(long_delta_df) == 1 + assert len(base_delta_df) == 1 + long_delta = long_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected longs + assert long_delta["delta"] == -actual_num_longs + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + if txn["input_method"] == "closeShort": + assert txn["input_params_bondAmount"] == Decimal(33333) + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + short_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "SHORT"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(short_delta_df) == 1 + assert len(base_delta_df) == 1 + short_delta = short_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected shorts + assert short_delta["delta"] == -actual_num_shorts + # TODO check base delta + # TODO check maturity time and tokenType + # TODO check wallet info matches the deltas + # TODO check pool info after this tx + + if txn["input_method"] == "redeemWithdrawalShares": + # TODO change this to expected withdrawal shares + assert txn["input_params_shares"] == actual_num_withdrawal + block_wallet_deltas = db_wallet_delta[db_wallet_delta["blockNumber"] == block_number] + assert len(block_wallet_deltas) == 2 + withdrawal_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "WITHDRAWAL_SHARE"] + base_delta_df = block_wallet_deltas[block_wallet_deltas["baseTokenType"] == "BASE"] + assert len(withdrawal_delta_df) == 1 + assert len(base_delta_df) == 1 + withdrawal_delta = withdrawal_delta_df.iloc[0] + base_delta = base_delta_df.iloc[0] + # TODO check against expected withdrawal shares + assert withdrawal_delta["delta"] == -actual_num_withdrawal + # TODO check base delta + # TODO check wallet info matches the deltas + # TODO check pool info after this tx From 32ce636db17dd400973b2753aff7c78af7f3a8bb Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 11:51:52 -0700 Subject: [PATCH 28/32] formatting changes --- tests/system_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/system_test.py b/tests/system_test.py index 90e491cda9..4a2cb3dfb4 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -17,7 +17,6 @@ from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch from fixedpointmath import FixedPoint from sqlalchemy.orm import Session -from web3 import Web3 # This pass is to prevent auto reordering imports from reordering the imports below pass # pylint: disable=unnecessary-pass @@ -28,7 +27,7 @@ AgentDoneException, cycle_trade_policy, ) -from chainsync.test_fixtures import db_session # pylint: disable=unused-import +from chainsync.test_fixtures import db_session # pylint: disable=unused-import, ungrouped-imports from ethpy.test_fixtures import local_chain, local_hyperdrive_chain # pylint: disable=unused-import, ungrouped-imports # fixture arguments in test function have to be the same as the fixture name @@ -50,8 +49,8 @@ def _to_unscaled_decimal(scaled_value: int) -> Decimal: return Decimal(str(FixedPoint(scaled_value=scaled_value))) -def _decimal_almost_equal(a: Decimal, b: Decimal) -> bool: - return abs(a - b) < 1e-12 +def _decimal_almost_equal(a_val: Decimal, b_val: Decimal) -> bool: + return abs(a_val - b_val) < 1e-12 class TestBotToDb: @@ -65,8 +64,10 @@ def test_bot_to_db( cycle_trade_policy: Type[BasePolicy], db_session: Session, ): + """Runs the entire pipeline and checks the database at the end. + All arguments are fixtures. + """ # Get hyperdrive chain info - web3: Web3 = local_hyperdrive_chain["web3"] deploy_account: LocalAccount = local_hyperdrive_chain["deploy_account"] hyperdrive_contract_addresses: HyperdriveAddresses = local_hyperdrive_chain["hyperdrive_contract_addresses"] From 55bf4642265a40c3c3436732a434a1ae683a3387 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 12:08:58 -0700 Subject: [PATCH 29/32] lint --- lib/chainsync/chainsync/db/hyperdrive/interface_test.py | 2 +- lib/chainsync/chainsync/exec/__init__.py | 1 + tests/system_test.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py index ced7a00a77..6c03599a06 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface_test.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface_test.py @@ -187,7 +187,7 @@ def test_pool_config_verify(self, db_session): # Nothing should happen if we give the same pool_config # TODO Below is a hack due to sqlite not having numerics # We explicitly print 18 spots after floating point to match rounding error in sqlite - pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal("{:.18f}".format(3.2))) + pool_config_2 = PoolConfig(contractAddress="0", initialSharePrice=Decimal(f"{3.2:.18f}")) add_pool_config(pool_config_2, db_session) pool_config_df_2 = get_pool_config(db_session) assert len(pool_config_df_2) == 1 diff --git a/lib/chainsync/chainsync/exec/__init__.py b/lib/chainsync/chainsync/exec/__init__.py index 80fd80d4d7..d79bc21bb0 100644 --- a/lib/chainsync/chainsync/exec/__init__.py +++ b/lib/chainsync/chainsync/exec/__init__.py @@ -1 +1,2 @@ +"""Execution functions for chainsync""" from .acquire_data import acquire_data diff --git a/tests/system_test.py b/tests/system_test.py index 4a2cb3dfb4..08a35358ee 100644 --- a/tests/system_test.py +++ b/tests/system_test.py @@ -57,6 +57,7 @@ class TestBotToDb: """Tests pipeline from bots making trades to viewing the trades in the db""" # TODO split this up into different functions that work with tests + # pylint: disable=too-many-locals, too-many-statements def test_bot_to_db( self, local_chain: str, From efda644562dd91ea59dd9062dfe8411d873dc7ac Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 12:28:11 -0700 Subject: [PATCH 30/32] Updating readme with foundary prereq --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e99d8678c5..8329e98bc4 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Please refer to [BUILD.md](https://github.com/delvtech/elf-simulations/blob/main ## Testing +We deploy a local anvil chain to run system tests. Therefore, you must [install foundry](https://github.com/foundry-rs/foundry#installatio://github.com/foundry-rs/foundry#installation) as a prerequisite for running tests. + Testing is achieved with [py.test](https://docs.pytest.org/en/latest/contents.html). You can run all tests from the repository root directory by running `python -m pytest`, or you can pick a specific test in the `tests/` folder with `python -m pytest tests/{test_file.py}`. ## Coverage From 55846bf3fc141914f022b9628f3bffa182493bc1 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 12:38:55 -0700 Subject: [PATCH 31/32] Adding comment for precision and scale --- lib/chainsync/chainsync/db/hyperdrive/schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/chainsync/chainsync/db/hyperdrive/schema.py b/lib/chainsync/chainsync/db/hyperdrive/schema.py index bf0f1684ee..3cbf57f437 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/schema.py +++ b/lib/chainsync/chainsync/db/hyperdrive/schema.py @@ -11,6 +11,8 @@ # pylint: disable=invalid-name # Postgres numeric type that matches fixedpoint +# Precision here indicates the total number of significant digits to store, +# while scale indicates the number of digits to the right of the decimal # The high precision doesn't actually allocate memory in postgres, as numeric is variable size # https://stackoverflow.com/questions/40686571/performance-of-numeric-type-with-high-precisions-and-scales-in-postgresql FIXED_NUMERIC = Numeric(precision=1000, scale=18) From 018885d081fc272d004162d09c6f7c787af1fcd4 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 17 Aug 2023 12:40:51 -0700 Subject: [PATCH 32/32] Updating todo comment --- lib/ethpy/ethpy/hyperdrive/interface.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/ethpy/ethpy/hyperdrive/interface.py b/lib/ethpy/ethpy/hyperdrive/interface.py index f677175af6..b98f6dae97 100644 --- a/lib/ethpy/ethpy/hyperdrive/interface.py +++ b/lib/ethpy/ethpy/hyperdrive/interface.py @@ -113,10 +113,9 @@ def get_hyperdrive_config(hyperdrive_contract: Contract) -> dict[str, Any]: pool_config["minimumShareReserves"] = FixedPoint(scaled_value=hyperdrive_config["minimumShareReserves"]) pool_config["positionDuration"] = hyperdrive_config["positionDuration"] pool_config["checkpointDuration"] = hyperdrive_config["checkpointDuration"] - # Ok so, the contracts store the time stretch constant in an inverted manner from the python. + # TODO Ok so, the contracts store the time stretch constant in an inverted manner from the python. # In order to not break the world, we save the contract version as 'invTimeStretch' and invert - # that to get the python version 'timeStretch' - # TODO fix this + # that to get the python version 'timeStretch'. Fix this issue to match solidity pool_config["invTimeStretch"] = FixedPoint(scaled_value=hyperdrive_config["timeStretch"]) pool_config["timeStretch"] = FixedPoint(1) / pool_config["invTimeStretch"] pool_config["governance"] = hyperdrive_config["governance"]