From c5751041f0ed5a7a54c3db0f22e69d41610ab70c Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:08:50 -0700 Subject: [PATCH 01/31] Generalizing register username server to database api server --- .../agent0/base/config/environment_config.py | 4 +- lib/agent0/examples/example_agent.py | 3 +- lib/agent0/examples/hyperdrive_agents.py | 1 + lib/chainsync/bin/register_username_server.py | 48 --------- lib/chainsync/bin/run_api_server.py | 25 +++++ lib/chainsync/chainsync/db/__init__.py | 1 + lib/chainsync/chainsync/db/api_server.py | 97 +++++++++++++++++++ .../chainsync/db/hyperdrive/interface.py | 12 ++- .../chainsync/test_fixtures/__init__.py | 2 +- 9 files changed, 137 insertions(+), 56 deletions(-) delete mode 100644 lib/chainsync/bin/register_username_server.py create mode 100644 lib/chainsync/bin/run_api_server.py create mode 100644 lib/chainsync/chainsync/db/api_server.py diff --git a/lib/agent0/agent0/base/config/environment_config.py b/lib/agent0/agent0/base/config/environment_config.py index 8320c52e0d..309e95d894 100644 --- a/lib/agent0/agent0/base/config/environment_config.py +++ b/lib/agent0/agent0/base/config/environment_config.py @@ -37,8 +37,8 @@ class EnvironmentConfig(types.FrozenClass): max_bytes: int = DEFAULT_LOG_MAXBYTES # int(2e6) or 2MB # int to be used for the random seed random_seed: int = 1 - # username registration URI - username_register_uri: str = "http://localhost:5002" + # Database api for username registration and wallet queries + database_api_uri: str = "http://localhost:5002" def __getitem__(self, attrib) -> None: return getattr(self, attrib) diff --git a/lib/agent0/examples/example_agent.py b/lib/agent0/examples/example_agent.py index f8c97e9f73..0a25154a62 100644 --- a/lib/agent0/examples/example_agent.py +++ b/lib/agent0/examples/example_agent.py @@ -153,7 +153,8 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis log_level=logging.INFO, log_stdout=True, random_seed=1234, - username="tmp", + database_api_uri="http://localhost:5002", + username="changeme", ) # Build agent config diff --git a/lib/agent0/examples/hyperdrive_agents.py b/lib/agent0/examples/hyperdrive_agents.py index a1fa64ea69..a2e6c64b64 100644 --- a/lib/agent0/examples/hyperdrive_agents.py +++ b/lib/agent0/examples/hyperdrive_agents.py @@ -25,6 +25,7 @@ log_level=logging.INFO, log_stdout=True, random_seed=1234, + database_api_uri="http://localhost:5002", username="changeme", ) diff --git a/lib/chainsync/bin/register_username_server.py b/lib/chainsync/bin/register_username_server.py deleted file mode 100644 index ed63279e11..0000000000 --- a/lib/chainsync/bin/register_username_server.py +++ /dev/null @@ -1,48 +0,0 @@ -"""A simple Flask server to run python scripts.""" -import logging - -from chainsync.db.base import interface -from dotenv import load_dotenv -from flask import Flask, jsonify, request -from flask_expects_json import expects_json - -app = Flask(__name__) - - -json_schema = { - "type": "object", - "properties": {"wallet_addrs": {"type": "array", "items": {"type": "string"}}, "username": {"type": "string"}}, - "required": ["wallet_addrs", "username"], -} - - -@app.route("/register_agents", methods=["POST"]) -@expects_json(json_schema) -def register_agents(): - """Registers a list of wallet addresses to a username via post request""" - # TODO: validate the json - data = request.json - if data is not None: - wallet_addrs: list[str] = data["wallet_addrs"] - username: str = data["username"] - else: - return jsonify({"data": data, "error": "request.json is None"}), 500 - - # initialize the postgres session - session = interface.initialize_session() - try: - interface.add_user_map(username, wallet_addrs, session) - logging.debug("Registered wallet_addrs=%s to username=%s}", wallet_addrs, username) - out = (jsonify({"data": data, "error": ""}), 200) - except Exception as exc: # pylint: disable=broad-exception-caught - # Ignoring broad exception, since we're simply printing out error and returning to client - out = (jsonify({"data": data, "error": str(exc)}), 500) - - interface.close_session(session) - return out - - -if __name__ == "__main__": - # Get postgres env variables if exists - load_dotenv() - app.run(host="0.0.0.0", port=5002) diff --git a/lib/chainsync/bin/run_api_server.py b/lib/chainsync/bin/run_api_server.py new file mode 100644 index 0000000000..c899b9129d --- /dev/null +++ b/lib/chainsync/bin/run_api_server.py @@ -0,0 +1,25 @@ +"""A simple Flask server to run python scripts.""" +import argparse + +from chainsync.db import launch_flask + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="Launches the database api server", + ) + parser.add_argument( + "--host", + nargs=1, + help="The hostname", + action="store", + default=[None], + ) + parser.add_argument( + "--port", + nargs=1, + help="The port", + action="store", + default=[None], + ) + args = parser.parse_args() + launch_flask(host=args.host[0], port=args.port[0]) diff --git a/lib/chainsync/chainsync/db/__init__.py b/lib/chainsync/chainsync/db/__init__.py index e69de29bb2..fba82b769b 100644 --- a/lib/chainsync/chainsync/db/__init__.py +++ b/lib/chainsync/chainsync/db/__init__.py @@ -0,0 +1 @@ +from .api_server import launch_flask diff --git a/lib/chainsync/chainsync/db/api_server.py b/lib/chainsync/chainsync/db/api_server.py new file mode 100644 index 0000000000..01f5fb4bb4 --- /dev/null +++ b/lib/chainsync/chainsync/db/api_server.py @@ -0,0 +1,97 @@ +"""A simple Flask server to run python scripts.""" +from __future__ import annotations + +import logging + +from chainsync.db.base import add_user_map, close_session, initialize_session +from chainsync.db.hyperdrive import get_current_wallet +from flask import Flask, jsonify, request +from flask_expects_json import expects_json + +app = Flask(__name__) + + +register_agents_json_schema = { + "type": "object", + "properties": {"wallet_addrs": {"type": "array", "items": {"type": "string"}}, "username": {"type": "string"}}, + "required": ["wallet_addrs", "username"], +} + + +@app.route("/register_agents", methods=["POST"]) +@expects_json(register_agents_json_schema) +def register_agents(): + """Registers a list of wallet addresses to a username via post request""" + # TODO: validate the json + data = request.json + if data is not None: + wallet_addrs: list[str] = data["wallet_addrs"] + username: str = data["username"] + else: + return jsonify({"data": data, "error": "request.json is None"}), 500 + + # initialize the postgres session + # This function gets env variables for db credentials + session = initialize_session() + try: + add_user_map(username, wallet_addrs, session) + logging.debug("Registered wallet_addrs=%s to username=%s}", wallet_addrs, username) + out = (jsonify({"data": data, "error": ""}), 200) + except Exception as exc: # pylint: disable=broad-exception-caught + # Ignoring broad exception, since we're simply printing out error and returning to client + out = (jsonify({"data": data, "error": str(exc)}), 500) + + close_session(session) + return out + + +balance_of_json_schema = { + "type": "object", + "properties": {"wallet_addrs": {"type": "array", "items": {"type": "string"}}}, + "required": ["wallet_addrs"], +} + + +@app.route("/balance_of", methods=["POST"]) +@expects_json(balance_of_json_schema) +def balance_of(): + """Retrieves the balance of a given wallet address from the db. + Note that this only takes into account token differences from opening and closing + longs and shorts, not any transfer events between wallets. + """ + # TODO: validate the json + data = request.json + if data is not None: + wallet_addrs: list[str] = data["wallet_addrs"] + else: + return jsonify({"data": data, "error": "request.json is None"}), 500 + + # initialize the postgres session + # This function gets env variables for db credentials + session = initialize_session() + try: + logging.debug("Querying wallet_addrs=%s for balances}", wallet_addrs) + current_wallet = get_current_wallet(session, wallet_address=wallet_addrs, coerce_float=False) + # Cast decimal to string, then convert to json and return + data = current_wallet.astype(str).to_json() + + # Convert dataframe to json + out = (jsonify({"data": data, "error": ""}), 200) + except Exception as exc: # pylint: disable=broad-exception-caught + # Ignoring broad exception, since we're simply printing out error and returning to client + out = (jsonify({"data": data, "error": str(exc)}), 500) + + close_session(session) + return out + + +def launch_flask(host: str = "0.0.0.0", port: int = 5002): + """Launches the flask server + + Arguments + --------- + db_session: Session | None + Session object for connecting to db. If None, will initialize a new session based on + postgres.env. + """ + app.run(host=host, port=port) diff --git a/lib/chainsync/chainsync/db/hyperdrive/interface.py b/lib/chainsync/chainsync/db/hyperdrive/interface.py index 76751cd3c4..f472e47442 100644 --- a/lib/chainsync/chainsync/db/hyperdrive/interface.py +++ b/lib/chainsync/chainsync/db/hyperdrive/interface.py @@ -598,19 +598,20 @@ def add_current_wallet(current_wallet: list[CurrentWallet], session: Session) -> raise err -def get_current_wallet(session: Session, end_block: int | None = None, coerce_float=True) -> pd.DataFrame: +def get_current_wallet( + session: Session, end_block: int | None = None, wallet_address: list[str] | None = None, coerce_float=True +) -> pd.DataFrame: """Get all current wallet data in history and returns as a pandas dataframe. Arguments --------- session : Session The initialized session object - start_block : int | None, optional - The starting block to filter the query on. start_block integers - matches python slicing notation, e.g., list[:3], list[:-3] end_block : int | None, optional The ending block to filter the query on. end_block integers matches python slicing notation, e.g., list[:3], list[:-3] + wallet_address : list[str] | None, optional + The wallet addresses to filter the query on coerce_float : bool If true, will return floats in dataframe. Otherwise, will return fixed point Decimal @@ -640,6 +641,9 @@ def get_current_wallet(session: Session, end_block: int | None = None, coerce_fl elif end_block < 0: end_block = get_latest_block_number_from_table(CurrentWallet, session) + end_block + 1 + if wallet_address is not None: + query = query.filter(CurrentWallet.walletAddress.in_(wallet_address)) + query = query.filter(CurrentWallet.blockNumber < end_block) query = query.distinct(CurrentWallet.walletAddress, CurrentWallet.tokenType) query = query.order_by(CurrentWallet.walletAddress, CurrentWallet.tokenType, CurrentWallet.blockNumber.desc()) diff --git a/lib/chainsync/chainsync/test_fixtures/__init__.py b/lib/chainsync/chainsync/test_fixtures/__init__.py index a490ab4d77..6d2e39af3f 100644 --- a/lib/chainsync/chainsync/test_fixtures/__init__.py +++ b/lib/chainsync/chainsync/test_fixtures/__init__.py @@ -1,3 +1,3 @@ """Test fixtures for chainsync""" -from .db_session import database_engine, db_session, psql_docker +from .db_session import database_engine, db_api, db_session, psql_docker from .dummy_session import dummy_session From 4c04cbecf1ab53e898c983761ccbbb5f6c6ba8a9 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:09:47 -0700 Subject: [PATCH 02/31] Adding balance of call to db api server --- lib/agent0/agent0/hyperdrive/exec/__init__.py | 2 +- .../agent0/hyperdrive/exec/run_agents.py | 15 ++++--- .../hyperdrive/exec/setup_experiment.py | 41 ++++++++++++++++++- lib/agent0/pyproject.toml | 3 +- lib/chainsync/chainsync/db/base/interface.py | 2 +- 5 files changed, 53 insertions(+), 10 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/__init__.py b/lib/agent0/agent0/hyperdrive/exec/__init__.py index 90778a5c76..7afb4e0dfe 100644 --- a/lib/agent0/agent0/hyperdrive/exec/__init__.py +++ b/lib/agent0/agent0/hyperdrive/exec/__init__.py @@ -12,5 +12,5 @@ from .fund_agents import fund_agents from .get_agent_accounts import get_agent_accounts from .run_agents import run_agents -from .setup_experiment import register_username, setup_experiment +from .setup_experiment import balance_of, register_username, setup_experiment from .trade_loop import get_wait_for_new_block, trade_if_new_block diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index 204dcbef56..614f4e7ae7 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -13,7 +13,7 @@ from .create_and_fund_user_account import create_and_fund_user_account from .fund_agents import fund_agents -from .setup_experiment import register_username, setup_experiment +from .setup_experiment import balance_of, register_username, setup_experiment from .trade_loop import trade_if_new_block @@ -72,17 +72,22 @@ def run_agents( eth_config, environment_config, agent_config, account_key_config, contract_addresses ) + wallet_addrs = [str(agent.checksum_address) for agent in agent_accounts] if not develop: + # Ignore this check if not develop if environment_config.username == DEFAULT_USERNAME: # Check for default name and exit if is default raise ValueError( "Default username detected, please update 'username' in " "lib/agent0/agent0/hyperdrive/config/runner_config.py" ) - # Set up postgres to write username to agent wallet addr - # initialize the postgres session - wallet_addrs = [str(agent.checksum_address) for agent in agent_accounts] - register_username(environment_config.username_register_uri, wallet_addrs, environment_config.username) + # Register wallet addresses to username + register_username(environment_config.database_api_uri, wallet_addrs, environment_config.username) + + # Get existing open positions from db + balances = balance_of(environment_config.database_api_uri, wallet_addrs) + + # TODO Set balances of wallets based on db last_executed_block = BlockNumber(0) while True: diff --git a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py index 9d3b32e608..7f7bd2b382 100644 --- a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py +++ b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py @@ -1,9 +1,12 @@ """Setup helper function for running eth agent experiments.""" from __future__ import annotations +import logging +import time from http import HTTPStatus import numpy as np +import pandas as pd import requests from agent0 import AccountKeyConfig from agent0.base.config import AgentConfig, EnvironmentConfig @@ -70,10 +73,11 @@ def setup_experiment( agent_accounts = get_agent_accounts( web3, agent_config, account_key_config, base_token_contract, hyperdrive_contract.address, rng ) + return web3, base_token_contract, hyperdrive_contract, agent_accounts -def register_username(register_uri: str, wallet_addrs: list[str], username: str) -> None: +def register_username(api_uri: str, wallet_addrs: list[str], username: str) -> None: """Registers the username with the flask server. Arguments @@ -87,6 +91,39 @@ def register_username(register_uri: str, wallet_addrs: list[str], username: str) """ # TODO: use the json schema from the server. json_data = {"wallet_addrs": wallet_addrs, "username": username} - result = requests.post(f"{register_uri}/register_agents", json=json_data, timeout=3) + result = requests.post(f"{api_uri}/register_agents", json=json_data, timeout=3) if result.status_code != HTTPStatus.OK: raise ConnectionError(result) + + +def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: + """Gets all open positions for a given list of wallet addresses from the db + + Arguments + --------- + : str + The endpoint for the flask server. + wallet_addrs: list[str] + The list of wallet addresses to register. + username: str + The username to register the wallet addresses under. + """ + # TODO: use the json schema from the server. + json_data = {"wallet_addrs": wallet_addrs} + result = None + for _ in range(10): + try: + result = requests.post(f"{api_uri}/balance_of", json=json_data, timeout=3) + except requests.exceptions.RequestException: + logging.warning("Connection error to db api server, retrying") + time.sleep(1) + continue + + if result is None or (result.status_code != HTTPStatus.OK): + raise ConnectionError(result) + + # Read json and return + # Since we use pandas write json, we use pandas read json to read, then adjust data + # before returning + data = pd.read_json(result.json()["data"]) + return data diff --git a/lib/agent0/pyproject.toml b/lib/agent0/pyproject.toml index 95ce1998f8..27cb1f6ef6 100644 --- a/lib/agent0/pyproject.toml +++ b/lib/agent0/pyproject.toml @@ -28,7 +28,8 @@ base = [ "requests", "python-dotenv", "web3", # will include eth- packages - "hexbytes" + "hexbytes", + "panads" ] lateral = [ # Lateral dependencies across subpackages are pointing to github diff --git a/lib/chainsync/chainsync/db/base/interface.py b/lib/chainsync/chainsync/db/base/interface.py index 35a91297a4..7305d6e0fc 100644 --- a/lib/chainsync/chainsync/db/base/interface.py +++ b/lib/chainsync/chainsync/db/base/interface.py @@ -85,7 +85,7 @@ def initialize_engine(postgres_config: PostgresConfig | None = None) -> Engine: exception = None break except OperationalError as ex: - logging.warning("No connection, retrying") + logging.warning("No postgres connection, retrying") exception = ex time.sleep(1) if exception is not None: From 2c123596a547133720437c31dee147caa1e9b4fc Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:10:10 -0700 Subject: [PATCH 03/31] Adjusting tests to take the now required db api server --- conftest.py | 3 +- .../chainsync/test_fixtures/db_session.py | 52 +++++++++++++++++-- tests/bot_to_db_test.py | 2 + 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/conftest.py b/conftest.py index dbd8f85c63..8b3884ec31 100644 --- a/conftest.py +++ b/conftest.py @@ -6,7 +6,7 @@ import pytest from agent0.test_fixtures import cycle_trade_policy -from chainsync.test_fixtures import database_engine, db_session, dummy_session, psql_docker +from chainsync.test_fixtures import database_engine, db_api, db_session, dummy_session, psql_docker from ethpy.test_fixtures import local_chain, local_hyperdrive_chain # Hack to allow for vscode debugger to throw exception immediately @@ -49,6 +49,7 @@ def pytest_internalerror(excinfo): # TODO this means pytest can only be ran from this directory __all__ = [ "database_engine", + "db_api", "db_session", "dummy_session", "psql_docker", diff --git a/lib/chainsync/chainsync/test_fixtures/db_session.py b/lib/chainsync/chainsync/test_fixtures/db_session.py index c821f6e4a2..7e1c7c950d 100644 --- a/lib/chainsync/chainsync/test_fixtures/db_session.py +++ b/lib/chainsync/chainsync/test_fixtures/db_session.py @@ -1,5 +1,8 @@ """Pytest fixture that creates an in memory db session and creates the base db schema""" +import os +import subprocess import time +from pathlib import Path from typing import Iterator import docker @@ -42,7 +45,7 @@ def psql_docker() -> Iterator[PostgresConfig]: ) # Wait for the container to start - time.sleep(5) + time.sleep(3) yield postgres_config @@ -58,7 +61,7 @@ def database_engine(psql_docker): postgres_config = psql_docker with DatabaseJanitor( user=postgres_config.POSTGRES_USER, - host="localhost", + host="127.0.0.1", port=postgres_config.POSTGRES_PORT, dbname=postgres_config.POSTGRES_DB, # TODO set to latest postgres (what's being used in infra) @@ -71,11 +74,54 @@ def database_engine(psql_docker): @pytest.fixture(scope="function") def db_session(database_engine) -> Iterator[Session]: - """Initializes the in memory db session and creates the db schema""" + """Initializes the in memory db session and creates the db schema + + Returns + ------- + Iterator[Tuple[Session, str]] + Yields the sqlalchemy session object, with the database api uri + """ + session = sessionmaker(bind=database_engine) Base.metadata.create_all(database_engine) # create tables db_session_ = session() + yield db_session_ + db_session_.close() Base.metadata.drop_all(database_engine) # drop tables + + +@pytest.fixture(scope="function") +def db_api(psql_docker) -> Iterator[str]: + """Launches a process for the db api + + Returns + ------- + Iterator[str] + Yields the sqlalchemy session object, with the database api uri + """ + # Launch the database api server here + db_api_host = "127.0.0.1" + db_api_port = 5005 + + api_server_path = Path(__file__).parent.joinpath("../db/api_server") + + # Modify an environment to set db credentials + env = os.environ.copy() + # Set all env variables from the psql docker config + env["POSTGRES_USER"] = psql_docker.POSTGRES_USER + env["POSTGRES_PASSWORD"] = psql_docker.POSTGRES_PASSWORD + env["POSTGRES_DB"] = psql_docker.POSTGRES_DB + env["POSTGRES_HOST"] = psql_docker.POSTGRES_HOST + env["POSTGRES_PORT"] = str(psql_docker.POSTGRES_PORT) + + # Pass db credentials via env vars + api_process = subprocess.Popen( + ["flask", "--app", api_server_path, "run", "--host", db_api_host, "--port", str(db_api_port)], env=env + ) + + yield "http://" + db_api_host + ":" + str(db_api_port) + + api_process.kill() diff --git a/tests/bot_to_db_test.py b/tests/bot_to_db_test.py index b04f5ecf4e..06f2191a64 100644 --- a/tests/bot_to_db_test.py +++ b/tests/bot_to_db_test.py @@ -45,6 +45,7 @@ def test_bot_to_db( local_hyperdrive_chain: LocalHyperdriveChain, cycle_trade_policy: Type[BasePolicy], db_session: Session, + db_api: str, ): """Runs the entire pipeline and checks the database at the end. All arguments are fixtures. @@ -63,6 +64,7 @@ def test_bot_to_db( log_level=logging.INFO, log_stdout=True, random_seed=1234, + database_api_uri=db_api, username="test", ) From 8c15ecf132a5dafe3d50420a9a85fec94b464fe3 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:42:09 -0700 Subject: [PATCH 04/31] Adding structure for new bot load state test --- tests/bot_load_state_test.py | 225 +++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 tests/bot_load_state_test.py diff --git a/tests/bot_load_state_test.py b/tests/bot_load_state_test.py new file mode 100644 index 0000000000..f50a634722 --- /dev/null +++ b/tests/bot_load_state_test.py @@ -0,0 +1,225 @@ +"""System test for end to end testing of elf-simulations""" +from __future__ import annotations + +import logging +from decimal import Decimal +from typing import Type, cast + +import numpy as np +import pandas as pd +from agent0 import build_account_key_config_from_agent_config +from agent0.base.config import AgentConfig, EnvironmentConfig +from agent0.base.policies import BasePolicy +from agent0.hyperdrive.agents import HyperdriveWallet +from agent0.hyperdrive.exec import run_agents +from agent0.hyperdrive.policies import HyperdrivePolicy +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from agent0.test_fixtures import AgentDoneException +from chainsync.db.hyperdrive.interface import ( + get_current_wallet, + get_pool_config, + get_pool_info, + get_transactions, + get_wallet_deltas, +) +from chainsync.exec import acquire_data, data_analysis +from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState +from elfpy.types import MarketType, Trade +from eth_account.signers.local import LocalAccount +from eth_typing import URI +from ethpy import EthConfig +from ethpy.hyperdrive.addresses import HyperdriveAddresses +from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch +from ethpy.test_fixtures.local_chain import LocalHyperdriveChain +from fixedpointmath import FixedPoint +from numpy.random._generator import Generator as NumpyGenerator +from sqlalchemy.orm import Session +from web3 import HTTPProvider + + +class WalletTestPolicy(HyperdrivePolicy): + """A agent that simply cycles through all trades""" + + # Using default parameters + def __init__( + self, + budget: FixedPoint, + rng: NumpyGenerator | None = None, + slippage_tolerance: FixedPoint | None = None, + rerun: bool = False, + ): + # We want to do a sequence of trades one at a time, so we keep an internal counter based on + # how many times `action` has been called. + self.counter = 0 + self.rerun = rerun + super().__init__(budget, rng, slippage_tolerance) + + def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMarketAction]]: + """This agent simply opens all trades for a fixed amount and closes them after, one at a time""" + action_list = [] + + if self.rerun: + # TODO assert wallet state is up to date + + # We want this bot to exit and crash after it's done the trades it needs to do + raise AgentDoneException("Bot done") + + if self.counter == 0: + # Add liquidity + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.ADD_LIQUIDITY, + trade_amount=FixedPoint(11111), + wallet=wallet, + ), + ) + ) + elif self.counter == 1: + # Open Long + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_LONG, + trade_amount=FixedPoint(22222), + wallet=wallet, + ), + ) + ) + elif self.counter == 2: + # Open Short + action_list.append( + Trade( + market_type=MarketType.HYPERDRIVE, + market_action=HyperdriveMarketAction( + action_type=HyperdriveActionType.OPEN_SHORT, + trade_amount=FixedPoint(33333), + wallet=wallet, + ), + ) + ) + else: + # We want this bot to exit and crash after it's done the trades it needs to do + raise AgentDoneException("Bot done") + self.counter += 1 + return action_list + + +class TestBotToDb: + """Tests pipeline from bots making trades to viewing the trades in the db""" + + # TODO split this up into different functions that work with tests + # pylint: disable=too-many-locals, too-many-statements + def test_bot_to_db( + self, + local_hyperdrive_chain: LocalHyperdriveChain, + db_session: Session, + db_api: str, + ): + """Runs the entire pipeline and checks the database at the end. + All arguments are fixtures. + """ + # Get hyperdrive chain info + uri: URI | None = cast(HTTPProvider, local_hyperdrive_chain.web3.provider).endpoint_uri + rpc_uri = uri if uri else URI("http://localhost:8545") + hyperdrive_contract_addresses: HyperdriveAddresses = local_hyperdrive_chain.hyperdrive_contract_addresses + + # Build environment config + env_config = EnvironmentConfig( + delete_previous_logs=False, + halt_on_errors=True, + log_filename="system_test", + log_level=logging.INFO, + log_stdout=True, + random_seed=1234, + database_api_uri=db_api, + username="test", + ) + + # Build agent config + agent_config: list[AgentConfig] = [ + AgentConfig( + policy=WalletTestPolicy, + number_of_agents=1, + slippage_tolerance=FixedPoint("0.0001"), + base_budget_wei=FixedPoint("1_000_000").scaled_value, # 1 million base + eth_budget_wei=FixedPoint("100").scaled_value, # 100 base + init_kwargs={"rerun": False}, + ), + ] + + # No need for random seed, this bot is deterministic + account_key_config = build_account_key_config_from_agent_config(agent_config) + + # Build custom eth config pointing to local test chain + eth_config = EthConfig( + # Artifacts_uri isn't used here, as we explicitly set addresses and passed to run_bots + artifacts_uri="not_used", + rpc_uri=rpc_uri, + # Using default abi dir + ) + + # Run bots + try: + run_agents( + env_config, + agent_config, + account_key_config, + develop=True, + eth_config=eth_config, + contract_addresses=hyperdrive_contract_addresses, + ) + except AgentDoneException: + # Using this exception to stop the agents, + # so this exception is expected on test pass + pass + + # Run acquire data to get data from chain to db + acquire_data( + start_block=8, # First 7 blocks are deploying hyperdrive, ignore + eth_config=eth_config, + db_session=db_session, + contract_addresses=hyperdrive_contract_addresses, + # Exit the script after catching up to the chain + exit_on_catch_up=True, + ) + + # Run data analysis to calculate various analysis values + data_analysis( + start_block=8, # First 7 blocks are deploying hyperdrive, ignore + eth_config=eth_config, + db_session=db_session, + contract_addresses=hyperdrive_contract_addresses, + # Exit the script after catching up to the chain + exit_on_catch_up=True, + ) + + # Run bots again, this time ensuring wallet is up to date + + # Build agent config + agent_config: list[AgentConfig] = [ + AgentConfig( + policy=WalletTestPolicy, + number_of_agents=1, + slippage_tolerance=FixedPoint("0.0001"), + base_budget_wei=FixedPoint("1_000_000").scaled_value, # 1 million base + eth_budget_wei=FixedPoint("100").scaled_value, # 100 base + init_kwargs={"rerun": True}, + ), + ] + + try: + run_agents( + env_config, + agent_config, + account_key_config, + develop=True, + eth_config=eth_config, + contract_addresses=hyperdrive_contract_addresses, + ) + except AgentDoneException: + # Using this exception to stop the agents, + # so this exception is expected on test pass + pass From 06296d2ea6340b44c850ef3b831845afbbda0abd Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:47:19 -0700 Subject: [PATCH 05/31] Adding function structure for building wallet positions from data --- .../agent0/hyperdrive/exec/run_agents.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index 614f4e7ae7..a301e3a009 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -5,11 +5,14 @@ import os import warnings +import pandas as pd from agent0 import AccountKeyConfig from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config +from ethpy.base import smart_contract_read from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_uri +from web3.contract.contract import Contract from .create_and_fund_user_account import create_and_fund_user_account from .fund_agents import fund_agents @@ -68,7 +71,7 @@ def run_agents( user_account, eth_config, account_key_config, contract_addresses ) # uses env variables created above as inputs - web3, _, hyperdrive_contract, agent_accounts = setup_experiment( + web3, base_contract, hyperdrive_contract, agent_accounts = setup_experiment( eth_config, environment_config, agent_config, account_key_config, contract_addresses ) @@ -84,10 +87,12 @@ def run_agents( # Register wallet addresses to username register_username(environment_config.database_api_uri, wallet_addrs, environment_config.username) - # Get existing open positions from db + # Get existing open positions from db api server balances = balance_of(environment_config.database_api_uri, wallet_addrs) # TODO Set balances of wallets based on db + for agent in agent_accounts: + build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) last_executed_block = BlockNumber(0) while True: @@ -98,3 +103,27 @@ def run_agents( environment_config.halt_on_errors, last_executed_block, ) + + +def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame, base_contract: Contract): + # Contract call to get base balance + base_amount = smart_contract_read(base_contract, "balanceOf", wallet_addr) + # TODO build base object + + # TODO We can also get lp and withdraw shares from chain? + wallet_balances = db_balances[db_balances["walletAddress"] == wallet_addr] + + # Get longs + long_balances = wallet_balances[wallet_balances["baseTokenType" == "LONG"]] + # TODO iterate through long balances and build wallet object + + short_balances = wallet_balances[wallet_balances["baseTokenType" == "SHORT"]] + # TODO iterate through short balances and build wallet object + + lp_balances = wallet_balances[wallet_balances["baseTokenType" == "LP"]] + assert len(lp_balances) <= 1 + # TODO Build LP balances object + + withdraw_balances = wallet_balances[wallet_balances["baseTokenType" == "WITHDRAWAL_SHARE"]] + assert len(withdraw_balances) <= 1 + # TODO Build withdraw share object From b64426560b3c33d026c29034566308dabed3f6eb Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Thu, 21 Sep 2023 17:51:32 -0700 Subject: [PATCH 06/31] fixing typo --- lib/agent0/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/pyproject.toml b/lib/agent0/pyproject.toml index 27cb1f6ef6..b20d69586d 100644 --- a/lib/agent0/pyproject.toml +++ b/lib/agent0/pyproject.toml @@ -29,7 +29,7 @@ base = [ "python-dotenv", "web3", # will include eth- packages "hexbytes", - "panads" + "pandas" ] lateral = [ # Lateral dependencies across subpackages are pointing to github From 1e1323decd572107e819c92706a01b9555e89cd7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 12:51:57 -0700 Subject: [PATCH 07/31] Moving some types from elfpy to agent0 --- lib/agent0/agent0/base/__init__.py | 1 + .../agent0/base/config/environment_config.py | 7 +- lib/agent0/agent0/base/state/market_state.py | 4 +- lib/agent0/agent0/base/types.py | 117 ++++++++++++++++++ .../hyperdrive/agents/hyperdrive_wallet.py | 80 ++++++++++-- .../hyperdrive/state/hyperdrive_actions.py | 4 +- .../state/hyperdrive_market_state.py | 4 +- .../agent0/hyperdrive/state/trade_result.py | 4 +- 8 files changed, 197 insertions(+), 24 deletions(-) create mode 100644 lib/agent0/agent0/base/types.py diff --git a/lib/agent0/agent0/base/__init__.py b/lib/agent0/agent0/base/__init__.py index e69de29bb2..78395df939 100644 --- a/lib/agent0/agent0/base/__init__.py +++ b/lib/agent0/agent0/base/__init__.py @@ -0,0 +1 @@ +from .types import FrozenClass, Quantity, TokenType, freezable diff --git a/lib/agent0/agent0/base/config/environment_config.py b/lib/agent0/agent0/base/config/environment_config.py index 309e95d894..1c6b347524 100644 --- a/lib/agent0/agent0/base/config/environment_config.py +++ b/lib/agent0/agent0/base/config/environment_config.py @@ -4,15 +4,16 @@ import json from dataclasses import dataclass -from elfpy import DEFAULT_LOG_LEVEL, DEFAULT_LOG_MAXBYTES, types +from agent0.base import FrozenClass, freezable +from elfpy import DEFAULT_LOG_LEVEL, DEFAULT_LOG_MAXBYTES from elfpy.utils import json as output_utils DEFAULT_USERNAME = "changeme" -@types.freezable(frozen=False, no_new_attribs=True) +@freezable(frozen=False, no_new_attribs=True) @dataclass -class EnvironmentConfig(types.FrozenClass): +class EnvironmentConfig(FrozenClass): """Parameters that can be set either locally or passed from docker.""" # lots of configs! diff --git a/lib/agent0/agent0/base/state/market_state.py b/lib/agent0/agent0/base/state/market_state.py index 479c75398f..ca5e737296 100644 --- a/lib/agent0/agent0/base/state/market_state.py +++ b/lib/agent0/agent0/base/state/market_state.py @@ -5,10 +5,10 @@ from dataclasses import dataclass import elfpy -from elfpy import types +from agent0.base import freezable -@types.freezable(frozen=False, no_new_attribs=False) +@freezable(frozen=False, no_new_attribs=False) @dataclass(kw_only=True) class BaseMarketState: r"""The state of an AMM.""" diff --git a/lib/agent0/agent0/base/types.py b/lib/agent0/agent0/base/types.py new file mode 100644 index 0000000000..62724f675a --- /dev/null +++ b/lib/agent0/agent0/base/types.py @@ -0,0 +1,117 @@ +"""Core types used across the repo""" +from __future__ import annotations # types will be strings by default in 3.11 + +from dataclasses import asdict, dataclass, is_dataclass, replace +from enum import Enum +from functools import wraps +from typing import Any, Type + +from fixedpointmath import FixedPoint + + +class FrozenClass: + """Config object with frozen attributes""" + + def freeze(self): + """Disallows changing existing members""" + return NotImplemented + + def disable_new_attribs(self): + """Disallows adding new members""" + return NotImplemented + + def astype(self, _new_type): + """Cast all member attributes to a new type""" + return NotImplemented + + @property + def dtypes(self): + """Return a dict listing name & type of each member variable""" + return NotImplemented + + +def freezable(frozen: bool = False, no_new_attribs: bool = False) -> Type: + r"""A wrapper that allows classes to be frozen, such that existing member attributes cannot be changed""" + + def decorator(cls: Type) -> Type: + # this decorator should only be placed atop a dataclass + if not is_dataclass(cls): + raise TypeError("The class must be a data class.") + + @wraps(wrapped=cls, updated=()) + class DecoratedFrozenClass(cls, FrozenClass): + """Subclass cls to enable freezing of attributes + + .. todo:: resolve why pyright cannot access member "freeze" when instantiated_class.freeze() is called + """ + + def __init__(self, *args, frozen=frozen, no_new_attribs=no_new_attribs, **kwargs) -> None: + super().__init__(*args, **kwargs) + super().__setattr__("frozen", frozen) + super().__setattr__("no_new_attribs", no_new_attribs) + + def __setattr__(self, attrib: str, value: Any) -> None: + if hasattr(self, attrib) and hasattr(self, "frozen") and getattr(self, "frozen"): + raise AttributeError(f"{self.__class__.__name__} is frozen, cannot change attribute '{attrib}'.") + if not hasattr(self, attrib) and hasattr(self, "no_new_attribs") and getattr(self, "no_new_attribs"): + raise AttributeError( + f"{self.__class__.__name__} has no_new_attribs set, cannot add attribute '{attrib}'." + ) + super().__setattr__(attrib, value) + + def freeze(self) -> None: + """disallows changing existing members""" + super().__setattr__("frozen", True) + + def disable_new_attribs(self) -> None: + """disallows adding new members""" + super().__setattr__("no_new_attribs", True) + + def astype(self, new_type): + """Cast all member attributes to a new type""" + new_data = {} + for attr_name, attr_value in asdict(self).items(): + try: + if isinstance(attr_value, list): + new_data[attr_name] = [new_type(val) for val in attr_value] + else: + new_data[attr_name] = new_type(attr_value) + self.__annotations__[attr_name] = new_type + except (ValueError, TypeError) as err: + raise TypeError( + f"unable to cast {attr_name=} of type {type(attr_value)=} to {new_type=}" + ) from err + # create a new instance of the data class with the updated + # attributes, rather than modifying the current instance in-place + return replace(self, **new_data) + + @property + def dtypes(self) -> dict[str, type]: + """Return a dict listing name & type of each member variable""" + dtypes_dict: dict[str, type] = {} + for attr_name, attr_value in asdict(self).items(): + dtypes_dict[attr_name] = type(attr_value) + return dtypes_dict + + # Set the name of the wrapped class to the name of the input class to preserve metadata + DecoratedFrozenClass.__name__ = cls.__name__ + return DecoratedFrozenClass + + return decorator + + +class TokenType(Enum): + r"""A type of token""" + + BASE = "base" + + +@dataclass +class Quantity: + r"""An amount with a unit""" + + amount: FixedPoint + unit: TokenType + + def __neg__(self): + return Quantity(amount=-self.amount, unit=self.unit) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index 7c08d7f08d..9de2e1d2ff 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -6,11 +6,75 @@ from dataclasses import dataclass, field from typing import Iterable +from agent0.base import Quantity, TokenType, freezable +from agent0.base.agents import EthWallet from fixedpointmath import FixedPoint -from agent0.base.agents import EthWallet -from elfpy.wallet.wallet import Long, Short -from elfpy.wallet.wallet_deltas import WalletDeltas + +@freezable() +@dataclass() +class WalletDeltas: + r"""Stores changes for an agent's wallet + + Arguments + ---------- + balance : Quantity + The base assets that held by the trader. + lp_tokens : FixedPoint + The LP tokens held by the trader. + longs : Dict[FixedPoint, Long] + The long positions held by the trader. + shorts : Dict[FixedPoint, Short] + The short positions held by the trader. + borrows : Dict[FixedPoint, Borrow] + The borrow positions held by the trader. + """ + # dataclasses can have many attributes + # pylint: disable=too-many-instance-attributes + + # fungible + balance: Quantity = field(default_factory=lambda: Quantity(amount=FixedPoint(0), unit=TokenType.BASE)) + # TODO: Support multiple typed balances: + # balance: Dict[TokenType, Quantity] = field(default_factory=dict) + lp_tokens: FixedPoint = FixedPoint(0) + # non-fungible (identified by key=mint_time, stored as dict) + longs: dict[FixedPoint, Long] = field(default_factory=dict) + shorts: dict[FixedPoint, Short] = field(default_factory=dict) + withdraw_shares: FixedPoint = FixedPoint(0) + + def copy(self) -> WalletDeltas: + """Returns a new copy of self""" + return WalletDeltas(**copy.deepcopy(self.__dict__)) + + +@dataclass +class Long: + r"""An open long position. + + Arguments + ---------- + balance : FixedPoint + The amount of bonds that the position is long. + + .. todo:: make balance a Quantity to enforce units + """ + + balance: FixedPoint # bonds + + +@dataclass +class Short: + r"""An open short position. + + Arguments + ---------- + balance : FixedPoint + The amount of bonds that the position is short. + open_share_price : FixedPoint + The share price at the time the short was opened. + """ + + balance: FixedPoint @dataclass(kw_only=True) @@ -90,17 +154,7 @@ def _update_shorts(self, shorts: Iterable[tuple[FixedPoint, Short]]) -> None: short, ) if maturity_time in self.shorts: # entry already exists for this maturity_time, so add to it - old_balance = self.shorts[maturity_time].balance self.shorts[maturity_time].balance += short.balance - # If the balance is positive, we are opening a short, therefore do a weighted - # mean for the open share price. - # This covers an edge case where two shorts are opened for the same account in the same block. - # If the balance is negative, we don't want to update the open_short_price. - if short.balance > FixedPoint(0): - old_share_price = self.shorts[maturity_time].open_share_price - self.shorts[maturity_time].open_share_price = ( - short.open_share_price * short.balance + old_share_price * old_balance - ) / (short.balance + old_balance) else: self.shorts.update({maturity_time: short}) if self.shorts[maturity_time].balance == FixedPoint(0): diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py index 9c44cfddeb..1b91167d76 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from enum import Enum -import elfpy.types as types +from agent0.base import freezable from agent0.hyperdrive.agents import HyperdriveWallet from elfpy.markets.base import BaseMarketAction from fixedpointmath import FixedPoint @@ -25,7 +25,7 @@ class HyperdriveActionType(Enum): REDEEM_WITHDRAW_SHARE = "redeem_withdraw_share" -@types.freezable(frozen=False, no_new_attribs=True) +@freezable(frozen=False, no_new_attribs=True) @dataclass class HyperdriveMarketAction(BaseMarketAction): r"""Market action specification""" diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py index d9597241e3..41359a19a4 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py @@ -4,12 +4,12 @@ import copy from dataclasses import dataclass, field +from agent0.base import freezable from agent0.base.state import BaseMarketState -from elfpy import types from fixedpointmath import FixedPoint -@types.freezable(frozen=False, no_new_attribs=False) +@freezable(frozen=False, no_new_attribs=False) @dataclass(kw_only=True) class HyperdriveMarketState(BaseMarketState): r"""The state of an AMM diff --git a/lib/agent0/agent0/hyperdrive/state/trade_result.py b/lib/agent0/agent0/hyperdrive/state/trade_result.py index aa9c44de69..bd668189be 100644 --- a/lib/agent0/agent0/hyperdrive/state/trade_result.py +++ b/lib/agent0/agent0/hyperdrive/state/trade_result.py @@ -4,12 +4,12 @@ from dataclasses import dataclass -from elfpy import types +from agent0.base import freezable from elfpy.markets.base import BaseMarketActionResult from fixedpointmath import FixedPoint -@types.freezable(frozen=True, no_new_attribs=True) +@freezable(frozen=True, no_new_attribs=True) @dataclass class HyperdriveActionResult(BaseMarketActionResult): r"""The result to a market of performing a trade""" From aa80baeabcedc9d2d46d6e7a987e257e35e578e7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 12:59:52 -0700 Subject: [PATCH 08/31] Using new agent0 walletdeltas --- lib/agent0/agent0/base/agents/eth_wallet.py | 7 +++++-- lib/agent0/agent0/hyperdrive/agents/__init__.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index 18c5b3415e..757848732d 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -4,14 +4,17 @@ import copy import logging from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any from elfpy import check_non_zero from elfpy.types import Quantity, TokenType -from elfpy.wallet.wallet_deltas import WalletDeltas from fixedpointmath import FixedPoint from hexbytes import HexBytes +if TYPE_CHECKING: + # TODO need a base wallet delta + from elfpy.wallet.wallet_deltas import WalletDeltas + @dataclass(kw_only=True) class EthWallet: diff --git a/lib/agent0/agent0/hyperdrive/agents/__init__.py b/lib/agent0/agent0/hyperdrive/agents/__init__.py index 97176703ff..be8d27838b 100644 --- a/lib/agent0/agent0/hyperdrive/agents/__init__.py +++ b/lib/agent0/agent0/hyperdrive/agents/__init__.py @@ -1,3 +1,3 @@ """Account and wallet with Hyperdrive specific parts""" from .hyperdrive_account import HyperdriveAgent -from .hyperdrive_wallet import HyperdriveWallet +from .hyperdrive_wallet import HyperdriveWallet, Long, Short, WalletDeltas From 60aa7ee19a8b2a82e954f726d44be9d738ec323f Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 13:00:29 -0700 Subject: [PATCH 09/31] Another update for WalletDeltas --- lib/agent0/agent0/base/agents/eth_wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index 757848732d..4e6adc362e 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: # TODO need a base wallet delta - from elfpy.wallet.wallet_deltas import WalletDeltas + from agent0.hyperdrive.agents import WalletDeltas @dataclass(kw_only=True) From 06d6305e24c7f8dd0ebd1cce27cf1e89833acfff Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 15:08:45 -0700 Subject: [PATCH 10/31] Type updates --- lib/agent0/agent0/base/agents/eth_agent.py | 3 ++- lib/agent0/agent0/base/agents/eth_wallet.py | 2 +- .../agent0/hyperdrive/agents/hyperdrive_account.py | 9 ++++++--- .../agent0/hyperdrive/agents/hyperdrive_wallet.py | 12 ++++++------ .../agent0/hyperdrive/exec/execute_agent_trades.py | 2 +- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/agent0/agent0/base/agents/eth_agent.py b/lib/agent0/agent0/base/agents/eth_agent.py index 503fa07029..84d74db507 100644 --- a/lib/agent0/agent0/base/agents/eth_agent.py +++ b/lib/agent0/agent0/base/agents/eth_agent.py @@ -3,8 +3,9 @@ from typing import Generic, TypeVar +from agent0.base import Quantity, TokenType from agent0.base.policies import BasePolicy, NoActionPolicy -from elfpy.types import Quantity, TokenType, Trade +from elfpy.types import Trade from eth_account.signers.local import LocalAccount from eth_typing import ChecksumAddress from hexbytes import HexBytes diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index 4e6adc362e..0cfb1d6854 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -6,8 +6,8 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from agent0.base import Quantity, TokenType from elfpy import check_non_zero -from elfpy.types import Quantity, TokenType from fixedpointmath import FixedPoint from hexbytes import HexBytes diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py index 011b8ec860..a70c1252ad 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py @@ -4,11 +4,13 @@ import logging from typing import Generic, TypeVar +from agent0.base import Quantity, TokenType from agent0.base.agents import EthAgent from agent0.base.policies import BasePolicy from elfpy.markets.hyperdrive import HyperdriveMarket, HyperdriveMarketAction, MarketActionType -from elfpy.types import MarketType, Quantity, TokenType, Trade +from elfpy.types import MarketType, Trade from eth_account.signers.local import LocalAccount +from fixedpointmath import FixedPoint from hexbytes import HexBytes from .hyperdrive_wallet import HyperdriveWallet @@ -92,7 +94,8 @@ def liquidation_trades(self) -> list[Trade[MarketAction]]: action_type=MarketActionType.CLOSE_LONG, trade_amount=long.balance, wallet=self.wallet, # type: ignore - maturity_time=maturity_time, + # TODO this should be in int + maturity_time=FixedPoint(maturity_time), ), ) ) @@ -107,7 +110,7 @@ def liquidation_trades(self) -> list[Trade[MarketAction]]: action_type=MarketActionType.CLOSE_SHORT, trade_amount=short.balance, wallet=self.wallet, # type: ignore - maturity_time=maturity_time, + maturity_time=FixedPoint(maturity_time), ), ) ) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index 9de2e1d2ff..e5a9e3feae 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -38,8 +38,8 @@ class WalletDeltas: # balance: Dict[TokenType, Quantity] = field(default_factory=dict) lp_tokens: FixedPoint = FixedPoint(0) # non-fungible (identified by key=mint_time, stored as dict) - longs: dict[FixedPoint, Long] = field(default_factory=dict) - shorts: dict[FixedPoint, Short] = field(default_factory=dict) + longs: dict[int, Long] = field(default_factory=dict) + shorts: dict[int, Short] = field(default_factory=dict) withdraw_shares: FixedPoint = FixedPoint(0) def copy(self) -> WalletDeltas: @@ -103,10 +103,10 @@ class HyperdriveWallet(EthWallet): # pylint: disable=too-many-instance-attributes lp_tokens: FixedPoint = FixedPoint(0) withdraw_shares: FixedPoint = FixedPoint(0) - longs: dict[FixedPoint, Long] = field(default_factory=dict) - shorts: dict[FixedPoint, Short] = field(default_factory=dict) + longs: dict[int, Long] = field(default_factory=dict) + shorts: dict[int, Short] = field(default_factory=dict) - def _update_longs(self, longs: Iterable[tuple[FixedPoint, Long]]) -> None: + def _update_longs(self, longs: Iterable[tuple[int, Long]]) -> None: """Helper internal function that updates the data about Longs contained in the Agent's Wallet. Arguments @@ -135,7 +135,7 @@ def _update_longs(self, longs: Iterable[tuple[FixedPoint, Long]]) -> None: if maturity_time in self.longs and self.longs[maturity_time].balance < FixedPoint(0): raise AssertionError(f"ERROR: Wallet balance should be >= 0, not {self.longs[maturity_time]}.") - def _update_shorts(self, shorts: Iterable[tuple[FixedPoint, Short]]) -> None: + def _update_shorts(self, shorts: Iterable[tuple[int, Short]]) -> None: """Helper internal function that updates the data about Shorts contained in the Agent's Wallet. Arguments diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index e964567bb0..51866ba630 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -7,12 +7,12 @@ from typing import TYPE_CHECKING, NoReturn import eth_utils +from agent0.hyperdrive.agents import WalletDeltas from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction from elfpy import types from elfpy.markets.hyperdrive import HyperdriveMarket from elfpy.types import Quantity, TokenType from elfpy.wallet.wallet import Long, Short -from elfpy.wallet.wallet_deltas import WalletDeltas from ethpy.base import ( UnknownBlockError, async_smart_contract_transact, From c2732a099812e21b643347dbf2639624cc8d20d0 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 15:09:49 -0700 Subject: [PATCH 11/31] Building wallet object --- .../agent0/hyperdrive/exec/run_agents.py | 40 +++++++++++++++---- .../hyperdrive/exec/setup_experiment.py | 5 ++- .../chainsync/test_fixtures/db_session.py | 2 +- tests/bot_load_state_test.py | 15 +------ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index a301e3a009..ffc3ce9d08 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -7,11 +7,15 @@ import pandas as pd from agent0 import AccountKeyConfig +from agent0.base import Quantity, TokenType from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig +from agent0.hyperdrive.agents import HyperdriveWallet from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config from ethpy.base import smart_contract_read from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_uri +from fixedpointmath import FixedPoint +from hexbytes import HexBytes from web3.contract.contract import Contract from .create_and_fund_user_account import create_and_fund_user_account @@ -90,8 +94,14 @@ def run_agents( # Get existing open positions from db api server balances = balance_of(environment_config.database_api_uri, wallet_addrs) - # TODO Set balances of wallets based on db + # Set balances of wallets based on db and chain for agent in agent_accounts: + # TODO is this the right location for this to happen? + # On one hand, doing it here makes sense because parameters such as db uri doesn't have to + # be passed in down all the function calls when wallets are initialized. + # On the other hand, we initialize empty wallets just to overwrite here. + # Keeping here for now for later discussion + # TODO maybe this should be optional? build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) last_executed_block = BlockNumber(0) @@ -107,23 +117,37 @@ def run_agents( def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame, base_contract: Contract): # Contract call to get base balance - base_amount = smart_contract_read(base_contract, "balanceOf", wallet_addr) - # TODO build base object + base_amount: dict[str, int] = smart_contract_read(base_contract, "balanceOf", wallet_addr) + # TODO do we need to do error checking here? + assert "value" in base_amount + base_obj = Quantity(amount=FixedPoint(scaled_value=base_amount["value"]), unit=TokenType.BASE) # TODO We can also get lp and withdraw shares from chain? wallet_balances = db_balances[db_balances["walletAddress"] == wallet_addr] # Get longs - long_balances = wallet_balances[wallet_balances["baseTokenType" == "LONG"]] + long_balances = wallet_balances[wallet_balances["baseTokenType"] == "LONG"] # TODO iterate through long balances and build wallet object - short_balances = wallet_balances[wallet_balances["baseTokenType" == "SHORT"]] + short_balances = wallet_balances[wallet_balances["baseTokenType"] == "SHORT"] # TODO iterate through short balances and build wallet object - lp_balances = wallet_balances[wallet_balances["baseTokenType" == "LP"]] + lp_balances = wallet_balances[wallet_balances["baseTokenType"] == "LP"] assert len(lp_balances) <= 1 - # TODO Build LP balances object + if len(lp_balances) == 0: + lp_obj = FixedPoint(0) + else: + lp_obj = FixedPoint(lp_balances.iloc[0]["value"]) - withdraw_balances = wallet_balances[wallet_balances["baseTokenType" == "WITHDRAWAL_SHARE"]] + withdraw_balances = wallet_balances[wallet_balances["baseTokenType"] == "WITHDRAWAL_SHARE"] assert len(withdraw_balances) <= 1 + if len(lp_balances) == 0: + withdraw_obj = FixedPoint(0) + else: + withdraw_obj = FixedPoint(withdraw_balances.iloc[0]["value"]) # TODO Build withdraw share object + + wallet = HyperdriveWallet( + address=HexBytes(wallet_addr), + balance=base_obj, + ) diff --git a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py index 7f7bd2b382..969202320b 100644 --- a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py +++ b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py @@ -114,6 +114,7 @@ def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: for _ in range(10): try: result = requests.post(f"{api_uri}/balance_of", json=json_data, timeout=3) + break except requests.exceptions.RequestException: logging.warning("Connection error to db api server, retrying") time.sleep(1) @@ -125,5 +126,7 @@ def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: # Read json and return # Since we use pandas write json, we use pandas read json to read, then adjust data # before returning - data = pd.read_json(result.json()["data"]) + # We explicitly set dtype to False to keep everything in string format + # to avoid loss of precision + data = pd.read_json(result.json()["data"], dtype=False) return data diff --git a/lib/chainsync/chainsync/test_fixtures/db_session.py b/lib/chainsync/chainsync/test_fixtures/db_session.py index 7e1c7c950d..f6822e6185 100644 --- a/lib/chainsync/chainsync/test_fixtures/db_session.py +++ b/lib/chainsync/chainsync/test_fixtures/db_session.py @@ -50,7 +50,7 @@ def psql_docker() -> Iterator[PostgresConfig]: yield postgres_config # Docker doesn't play nice with types - container.stop() # type:ignore + container.kill() # type:ignore @pytest.fixture(scope="session") diff --git a/tests/bot_load_state_test.py b/tests/bot_load_state_test.py index f50a634722..f25a50cdc9 100644 --- a/tests/bot_load_state_test.py +++ b/tests/bot_load_state_test.py @@ -2,34 +2,21 @@ from __future__ import annotations import logging -from decimal import Decimal -from typing import Type, cast +from typing import cast -import numpy as np -import pandas as pd from agent0 import build_account_key_config_from_agent_config from agent0.base.config import AgentConfig, EnvironmentConfig -from agent0.base.policies import BasePolicy from agent0.hyperdrive.agents import HyperdriveWallet from agent0.hyperdrive.exec import run_agents from agent0.hyperdrive.policies import HyperdrivePolicy from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction from agent0.test_fixtures import AgentDoneException -from chainsync.db.hyperdrive.interface import ( - get_current_wallet, - get_pool_config, - get_pool_info, - get_transactions, - get_wallet_deltas, -) from chainsync.exec import acquire_data, data_analysis from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState from elfpy.types import MarketType, Trade -from eth_account.signers.local import LocalAccount from eth_typing import URI from ethpy import EthConfig from ethpy.hyperdrive.addresses import HyperdriveAddresses -from ethpy.test_fixtures.deploy_hyperdrive import _calculateTimeStretch from ethpy.test_fixtures.local_chain import LocalHyperdriveChain from fixedpointmath import FixedPoint from numpy.random._generator import Generator as NumpyGenerator From 18cb1b60ce775405fed61f8779a16b9518f43e85 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 15:22:17 -0700 Subject: [PATCH 12/31] No more mint time, maturity time instead. Maturity time as int. --- .../hyperdrive/agents/hyperdrive_wallet.py | 2 +- .../hyperdrive/exec/execute_agent_trades.py | 27 +++++++++---------- .../hyperdrive/policies/random_agent.py | 8 +++--- .../agent0/hyperdrive/policies/smart_long.py | 3 +-- .../agent0/hyperdrive/policies/smart_short.py | 3 +-- .../hyperdrive/state/hyperdrive_actions.py | 4 +-- .../state/hyperdrive_market_state.py | 12 ++++++--- .../test_fixtures/cycle_trade_policy.py | 5 ++-- lib/agent0/examples/example_agent.py | 5 ++-- 9 files changed, 32 insertions(+), 37 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index e5a9e3feae..caa8ddb86c 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -37,7 +37,7 @@ class WalletDeltas: # TODO: Support multiple typed balances: # balance: Dict[TokenType, Quantity] = field(default_factory=dict) lp_tokens: FixedPoint = FixedPoint(0) - # non-fungible (identified by key=mint_time, stored as dict) + # non-fungible (identified by key=maturity_time, stored as dict) longs: dict[int, Long] = field(default_factory=dict) shorts: dict[int, Short] = field(default_factory=dict) withdraw_shares: FixedPoint = FixedPoint(0) diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index 51866ba630..c8a3dc3cef 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -7,12 +7,11 @@ from typing import TYPE_CHECKING, NoReturn import eth_utils -from agent0.hyperdrive.agents import WalletDeltas +from agent0.base import Quantity, TokenType +from agent0.hyperdrive.agents import Long, Short, WalletDeltas from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction from elfpy import types from elfpy.markets.hyperdrive import HyperdriveMarket -from elfpy.types import Quantity, TokenType -from elfpy.wallet.wallet import Long, Short from ethpy.base import ( UnknownBlockError, async_smart_contract_transact, @@ -259,13 +258,13 @@ async def async_match_contract_call_to_trade( amount=-trade_result.base_amount, unit=TokenType.BASE, ), - longs={FixedPoint(maturity_time_seconds): Long(trade_result.bond_amount)}, + longs={maturity_time_seconds: Long(trade_result.bond_amount)}, ) case HyperdriveActionType.CLOSE_LONG: - if not trade.mint_time: - raise ValueError("Mint time was not provided, can't close long position.") - maturity_time_seconds = int(trade.mint_time) + if not trade.maturity_time: + raise ValueError("Maturity time was not provided, can't close long position.") + maturity_time_seconds = int(trade.maturity_time) min_output = 0 fn_args = ( maturity_time_seconds, @@ -294,7 +293,7 @@ async def async_match_contract_call_to_trade( amount=trade_result.base_amount, unit=TokenType.BASE, ), - longs={trade.mint_time: Long(-trade_result.bond_amount)}, + longs={trade.maturity_time: Long(-trade_result.bond_amount)}, ) case HyperdriveActionType.OPEN_SHORT: @@ -323,17 +322,16 @@ async def async_match_contract_call_to_trade( unit=TokenType.BASE, ), shorts={ - FixedPoint(maturity_time_seconds): Short( + maturity_time_seconds: Short( balance=trade_result.bond_amount, - open_share_price=hyperdrive_market.market_state.share_price, ) }, ) case HyperdriveActionType.CLOSE_SHORT: - if not trade.mint_time: - raise ValueError("Mint time was not provided, can't close long position.") - maturity_time_seconds = int(trade.mint_time) + if not trade.maturity_time: + raise ValueError("Maturity time was not provided, can't close long position.") + maturity_time_seconds = int(trade.maturity_time) min_output = 0 fn_args = ( maturity_time_seconds, @@ -363,9 +361,8 @@ async def async_match_contract_call_to_trade( unit=TokenType.BASE, ), shorts={ - trade.mint_time: Short( + trade.maturity_time: Short( balance=-trade_result.bond_amount, - open_share_price=agent.wallet.shorts[trade.mint_time].open_share_price, ) }, ) diff --git a/lib/agent0/agent0/hyperdrive/policies/random_agent.py b/lib/agent0/agent0/hyperdrive/policies/random_agent.py index 42995ceacd..263b052482 100644 --- a/lib/agent0/agent0/hyperdrive/policies/random_agent.py +++ b/lib/agent0/agent0/hyperdrive/policies/random_agent.py @@ -92,7 +92,7 @@ def open_short_with_random_amount(self, market: HyperdriveMarketState, wallet: H def close_random_short(self, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMarketAction]]: """Fully close the short balance for a random mint time.""" # choose a random short time to close - short_time: FixedPoint = list(wallet.shorts)[self.rng.integers(len(wallet.shorts))] + short_time = list(wallet.shorts)[self.rng.integers(len(wallet.shorts))] trade_amount = wallet.shorts[short_time].balance # close the full trade return [ Trade( @@ -102,7 +102,7 @@ def close_random_short(self, wallet: HyperdriveWallet) -> list[Trade[HyperdriveM trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=short_time, + maturity_time=short_time, ), ) ] @@ -136,7 +136,7 @@ def open_long_with_random_amount( def close_random_long(self, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMarketAction]]: """Fully close the long balance for a random mint time.""" # choose a random long time to close - long_time: FixedPoint = list(wallet.longs)[self.rng.integers(len(wallet.longs))] + long_time = list(wallet.longs)[self.rng.integers(len(wallet.longs))] trade_amount = wallet.longs[long_time].balance # close the full trade return [ Trade( @@ -146,7 +146,7 @@ def close_random_long(self, wallet: HyperdriveWallet) -> list[Trade[HyperdriveMa trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=long_time, + maturity_time=long_time, ), ) ] diff --git a/lib/agent0/agent0/hyperdrive/policies/smart_long.py b/lib/agent0/agent0/hyperdrive/policies/smart_long.py index c64594f1df..858a524c58 100644 --- a/lib/agent0/agent0/hyperdrive/policies/smart_long.py +++ b/lib/agent0/agent0/hyperdrive/policies/smart_long.py @@ -82,7 +82,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=long_time, + maturity_time=long_time, ), ) ] @@ -121,7 +121,6 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=market.block_time.time, ), ) ] diff --git a/lib/agent0/agent0/hyperdrive/policies/smart_short.py b/lib/agent0/agent0/hyperdrive/policies/smart_short.py index 6849c5398f..9e8cb45157 100644 --- a/lib/agent0/agent0/hyperdrive/policies/smart_short.py +++ b/lib/agent0/agent0/hyperdrive/policies/smart_short.py @@ -83,7 +83,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=short_time, + maturity_time=short_time, ), ) ] @@ -102,7 +102,6 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=trade_amount, slippage_tolerance=self.slippage_tolerance, wallet=wallet, - mint_time=market.block_time.time, ), ) ] diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py index 1b91167d76..0bfb24f12b 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py @@ -37,7 +37,5 @@ class HyperdriveMarketAction(BaseMarketAction): wallet: HyperdriveWallet # slippage tolerance percent where 0.01 would be a 1% tolerance slippage_tolerance: FixedPoint | None = None - # mint time is set only for trades that act on existing positions (close long or close short) - mint_time: FixedPoint | None = None # maturity time is set only for trades that act on existing positions (close long or close short) - maturity_time: FixedPoint | None = None + maturity_time: int | None = None diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py index 41359a19a4..6c8f70e10d 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_market_state.py @@ -123,10 +123,14 @@ def apply_delta(self, delta: HyperdriveMarketState) -> None: self.withdraw_capital += delta.withdraw_capital self.withdraw_interest += delta.withdraw_interest # checkpointing - for mint_time, delta_supply in delta.total_supply_longs.items(): - self.total_supply_longs[mint_time] = self.total_supply_longs.get(mint_time, FixedPoint(0)) + delta_supply - for mint_time, delta_supply in delta.total_supply_shorts.items(): - self.total_supply_shorts[mint_time] = self.total_supply_shorts.get(mint_time, FixedPoint(0)) + delta_supply + for maturity_time, delta_supply in delta.total_supply_longs.items(): + self.total_supply_longs[maturity_time] = ( + self.total_supply_longs.get(maturity_time, FixedPoint(0)) + delta_supply + ) + for maturity_time, delta_supply in delta.total_supply_shorts.items(): + self.total_supply_shorts[maturity_time] = ( + self.total_supply_shorts.get(maturity_time, FixedPoint(0)) + delta_supply + ) def copy(self) -> HyperdriveMarketState: """Returns a new copy of self""" diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py index 07a63fa4ce..31459fdc6a 100644 --- a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -104,8 +104,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis action_type=HyperdriveActionType.CLOSE_LONG, trade_amount=long.balance, wallet=wallet, - # TODO is this actually maturity time? Not mint time? - mint_time=long_time, + maturity_time=long_time, ), ) ) @@ -121,7 +120,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=short.balance, wallet=wallet, # TODO is this actually maturity time? Not mint time? - mint_time=short_time, + maturity_time=short_time, ), ) ) diff --git a/lib/agent0/examples/example_agent.py b/lib/agent0/examples/example_agent.py index 0a25154a62..0af34e19cc 100644 --- a/lib/agent0/examples/example_agent.py +++ b/lib/agent0/examples/example_agent.py @@ -107,8 +107,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis action_type=HyperdriveActionType.CLOSE_LONG, trade_amount=long.balance, wallet=wallet, - # TODO is this actually maturity time? Not mint time? - mint_time=long_time, + maturity_time=long_time, ), ) ) @@ -124,7 +123,7 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis trade_amount=short.balance, wallet=wallet, # TODO is this actually maturity time? Not mint time? - mint_time=short_time, + maturity_time=short_time, ), ) ) From ab5be9acab6fcca929eac84a932cdb90146cbb91 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 15:47:04 -0700 Subject: [PATCH 13/31] Setting agent wallet to loaded wallet and updating test --- .../hyperdrive/agents/hyperdrive_wallet.py | 2 - .../agent0/hyperdrive/exec/run_agents.py | 43 ++++++++++++++++--- tests/bot_load_state_test.py | 8 +++- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index caa8ddb86c..1fefa31ab2 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -70,8 +70,6 @@ class Short: ---------- balance : FixedPoint The amount of bonds that the position is short. - open_share_price : FixedPoint - The share price at the time the short was opened. """ balance: FixedPoint diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index ffc3ce9d08..e03d2c613e 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -9,7 +9,7 @@ from agent0 import AccountKeyConfig from agent0.base import Quantity, TokenType from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig -from agent0.hyperdrive.agents import HyperdriveWallet +from agent0.hyperdrive.agents import HyperdriveWallet, Long, Short from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config from ethpy.base import smart_contract_read @@ -91,9 +91,9 @@ def run_agents( # Register wallet addresses to username register_username(environment_config.database_api_uri, wallet_addrs, environment_config.username) + # Load existing balances # Get existing open positions from db api server balances = balance_of(environment_config.database_api_uri, wallet_addrs) - # Set balances of wallets based on db and chain for agent in agent_accounts: # TODO is this the right location for this to happen? @@ -102,7 +102,8 @@ def run_agents( # On the other hand, we initialize empty wallets just to overwrite here. # Keeping here for now for later discussion # TODO maybe this should be optional? - build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) + wallet = build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) + agent.wallet = wallet last_executed_block = BlockNumber(0) while True: @@ -115,7 +116,25 @@ def run_agents( ) -def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame, base_contract: Contract): +def build_wallet_positions_from_data( + wallet_addr: str, db_balances: pd.DataFrame, base_contract: Contract +) -> HyperdriveWallet: + """Builds a wallet position based on gathered data + + Arguments + --------- + wallet_addr: str + The checksum wallet address + db_balances: pd.DataFrame + The current positions dataframe gathered from the db (from the `balance_of` api call) + base_contract: Contract + The base contract to query the base amount from + + Returns + ------- + HyperdriveWallet + The wallet object build from the provided data + """ # Contract call to get base balance base_amount: dict[str, int] = smart_contract_read(base_contract, "balanceOf", wallet_addr) # TODO do we need to do error checking here? @@ -127,10 +146,14 @@ def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame # Get longs long_balances = wallet_balances[wallet_balances["baseTokenType"] == "LONG"] - # TODO iterate through long balances and build wallet object + long_obj = {} + for _, row in long_balances.iterrows(): + long_obj[row["maturityTime"]] = Long(balance=FixedPoint(row["value"])) short_balances = wallet_balances[wallet_balances["baseTokenType"] == "SHORT"] - # TODO iterate through short balances and build wallet object + short_obj = {} + for _, row in short_balances.iterrows(): + short_obj[row["maturityTime"]] = Short(balance=FixedPoint(row["value"])) lp_balances = wallet_balances[wallet_balances["baseTokenType"] == "LP"] assert len(lp_balances) <= 1 @@ -141,7 +164,7 @@ def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame withdraw_balances = wallet_balances[wallet_balances["baseTokenType"] == "WITHDRAWAL_SHARE"] assert len(withdraw_balances) <= 1 - if len(lp_balances) == 0: + if len(withdraw_balances) == 0: withdraw_obj = FixedPoint(0) else: withdraw_obj = FixedPoint(withdraw_balances.iloc[0]["value"]) @@ -150,4 +173,10 @@ def build_wallet_positions_from_data(wallet_addr: str, db_balances: pd.DataFrame wallet = HyperdriveWallet( address=HexBytes(wallet_addr), balance=base_obj, + lp_tokens=lp_obj, + withdraw_shares=withdraw_obj, + longs=long_obj, + shorts=short_obj, ) + + return wallet diff --git a/tests/bot_load_state_test.py b/tests/bot_load_state_test.py index f25a50cdc9..2c6accde3a 100644 --- a/tests/bot_load_state_test.py +++ b/tests/bot_load_state_test.py @@ -46,7 +46,13 @@ def action(self, market: HyperdriveMarketState, wallet: HyperdriveWallet) -> lis action_list = [] if self.rerun: - # TODO assert wallet state is up to date + # assert wallet state was loaded from previous run + assert len(wallet.longs) == 1 + assert len(wallet.shorts) == 1 + # TODO would like to check long and lp value here, + # but the units there are in bonds and lp shares respectively, + # where the known value of the trade is in units of base. + assert wallet.shorts[list(wallet.shorts.keys())[0]].balance == FixedPoint(33333) # We want this bot to exit and crash after it's done the trades it needs to do raise AgentDoneException("Bot done") From 4a607594dc59257c9588ca42959af80e63e826a7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 15:53:01 -0700 Subject: [PATCH 14/31] lint --- lib/agent0/agent0/base/__init__.py | 1 + lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py | 2 -- lib/agent0/agent0/hyperdrive/exec/run_agents.py | 7 ++----- lib/chainsync/chainsync/db/__init__.py | 1 + lib/chainsync/chainsync/test_fixtures/db_session.py | 2 ++ 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/agent0/agent0/base/__init__.py b/lib/agent0/agent0/base/__init__.py index 78395df939..eb6e3cb0aa 100644 --- a/lib/agent0/agent0/base/__init__.py +++ b/lib/agent0/agent0/base/__init__.py @@ -1 +1,2 @@ +"""The agent0 base module exports some types here""" from .types import FrozenClass, Quantity, TokenType, freezable diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index c8a3dc3cef..4079634d98 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -156,7 +156,6 @@ async def async_execute_single_agent_trade( wallet_deltas = await async_match_contract_call_to_trade( web3, hyperdrive_contract, - hyperdrive_market, agent, trade_object, ) @@ -194,7 +193,6 @@ async def async_execute_agent_trades( async def async_match_contract_call_to_trade( web3: Web3, hyperdrive_contract: Contract, - hyperdrive_market: HyperdriveMarket, agent: HyperdriveAgent, trade_envelope: types.Trade[HyperdriveMarketAction], ) -> WalletDeltas: diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index e03d2c613e..40f7beaf23 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -102,8 +102,7 @@ def run_agents( # On the other hand, we initialize empty wallets just to overwrite here. # Keeping here for now for later discussion # TODO maybe this should be optional? - wallet = build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) - agent.wallet = wallet + agent.wallet = build_wallet_positions_from_data(agent.checksum_address, balances, base_contract) last_executed_block = BlockNumber(0) while True: @@ -170,7 +169,7 @@ def build_wallet_positions_from_data( withdraw_obj = FixedPoint(withdraw_balances.iloc[0]["value"]) # TODO Build withdraw share object - wallet = HyperdriveWallet( + return HyperdriveWallet( address=HexBytes(wallet_addr), balance=base_obj, lp_tokens=lp_obj, @@ -178,5 +177,3 @@ def build_wallet_positions_from_data( longs=long_obj, shorts=short_obj, ) - - return wallet diff --git a/lib/chainsync/chainsync/db/__init__.py b/lib/chainsync/chainsync/db/__init__.py index fba82b769b..75cc638cf5 100644 --- a/lib/chainsync/chainsync/db/__init__.py +++ b/lib/chainsync/chainsync/db/__init__.py @@ -1 +1,2 @@ +"""Api server for the chainsync database.""" from .api_server import launch_flask diff --git a/lib/chainsync/chainsync/test_fixtures/db_session.py b/lib/chainsync/chainsync/test_fixtures/db_session.py index f6822e6185..5ebf3fc2f6 100644 --- a/lib/chainsync/chainsync/test_fixtures/db_session.py +++ b/lib/chainsync/chainsync/test_fixtures/db_session.py @@ -118,6 +118,8 @@ def db_api(psql_docker) -> Iterator[str]: env["POSTGRES_PORT"] = str(psql_docker.POSTGRES_PORT) # Pass db credentials via env vars + # Since this is a forever running service, we explicitly kill after the yield returns + # pylint: disable=consider-using-with api_process = subprocess.Popen( ["flask", "--app", api_server_path, "run", "--host", db_api_host, "--port", str(db_api_port)], env=env ) From 1b5b87d3e139e77dbb1ed2903900942e0112f5b2 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 16:09:04 -0700 Subject: [PATCH 15/31] Moving db api things to chainsync/db/api --- lib/agent0/agent0/hyperdrive/exec/__init__.py | 2 +- .../agent0/hyperdrive/exec/run_agents.py | 6 +- .../hyperdrive/exec/setup_experiment.py | 61 ------------------ lib/agent0/pyproject.toml | 1 - lib/chainsync/bin/run_api_server.py | 2 +- lib/chainsync/chainsync/db/__init__.py | 2 - lib/chainsync/chainsync/db/api/__init__.py | 3 + .../chainsync/db/api/api_interface.py | 62 +++++++++++++++++++ .../db/{api_server.py => api/flask_server.py} | 0 lib/chainsync/pyproject.toml | 1 + 10 files changed, 72 insertions(+), 68 deletions(-) create mode 100644 lib/chainsync/chainsync/db/api/__init__.py create mode 100644 lib/chainsync/chainsync/db/api/api_interface.py rename lib/chainsync/chainsync/db/{api_server.py => api/flask_server.py} (100%) diff --git a/lib/agent0/agent0/hyperdrive/exec/__init__.py b/lib/agent0/agent0/hyperdrive/exec/__init__.py index 7afb4e0dfe..39338ad96b 100644 --- a/lib/agent0/agent0/hyperdrive/exec/__init__.py +++ b/lib/agent0/agent0/hyperdrive/exec/__init__.py @@ -12,5 +12,5 @@ from .fund_agents import fund_agents from .get_agent_accounts import get_agent_accounts from .run_agents import run_agents -from .setup_experiment import balance_of, register_username, setup_experiment +from .setup_experiment import setup_experiment from .trade_loop import get_wait_for_new_block, trade_if_new_block diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index 40f7beaf23..53295500c4 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -10,17 +10,19 @@ from agent0.base import Quantity, TokenType from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig from agent0.hyperdrive.agents import HyperdriveWallet, Long, Short +from chainsync.db.api import balance_of, register_username from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config from ethpy.base import smart_contract_read -from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_uri +from ethpy.hyperdrive import (HyperdriveAddresses, + fetch_hyperdrive_address_from_uri) from fixedpointmath import FixedPoint from hexbytes import HexBytes from web3.contract.contract import Contract from .create_and_fund_user_account import create_and_fund_user_account from .fund_agents import fund_agents -from .setup_experiment import balance_of, register_username, setup_experiment +from .setup_experiment import setup_experiment from .trade_loop import trade_if_new_block diff --git a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py index 969202320b..db0a945cb5 100644 --- a/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py +++ b/lib/agent0/agent0/hyperdrive/exec/setup_experiment.py @@ -1,13 +1,7 @@ """Setup helper function for running eth agent experiments.""" from __future__ import annotations -import logging -import time -from http import HTTPStatus - import numpy as np -import pandas as pd -import requests from agent0 import AccountKeyConfig from agent0.base.config import AgentConfig, EnvironmentConfig from agent0.hyperdrive.agents import HyperdriveAgent @@ -75,58 +69,3 @@ def setup_experiment( ) return web3, base_token_contract, hyperdrive_contract, agent_accounts - - -def register_username(api_uri: str, wallet_addrs: list[str], username: str) -> None: - """Registers the username with the flask server. - - Arguments - --------- - register_uri: str - The endpoint for the flask server. - wallet_addrs: list[str] - The list of wallet addresses to register. - username: str - The username to register the wallet addresses under. - """ - # TODO: use the json schema from the server. - json_data = {"wallet_addrs": wallet_addrs, "username": username} - result = requests.post(f"{api_uri}/register_agents", json=json_data, timeout=3) - if result.status_code != HTTPStatus.OK: - raise ConnectionError(result) - - -def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: - """Gets all open positions for a given list of wallet addresses from the db - - Arguments - --------- - : str - The endpoint for the flask server. - wallet_addrs: list[str] - The list of wallet addresses to register. - username: str - The username to register the wallet addresses under. - """ - # TODO: use the json schema from the server. - json_data = {"wallet_addrs": wallet_addrs} - result = None - for _ in range(10): - try: - result = requests.post(f"{api_uri}/balance_of", json=json_data, timeout=3) - break - except requests.exceptions.RequestException: - logging.warning("Connection error to db api server, retrying") - time.sleep(1) - continue - - if result is None or (result.status_code != HTTPStatus.OK): - raise ConnectionError(result) - - # Read json and return - # Since we use pandas write json, we use pandas read json to read, then adjust data - # before returning - # We explicitly set dtype to False to keep everything in string format - # to avoid loss of precision - data = pd.read_json(result.json()["data"], dtype=False) - return data diff --git a/lib/agent0/pyproject.toml b/lib/agent0/pyproject.toml index b20d69586d..27064d3cbf 100644 --- a/lib/agent0/pyproject.toml +++ b/lib/agent0/pyproject.toml @@ -25,7 +25,6 @@ base = [ # TODO move fixedpointmath to lateral if this package comes back to this monorepo "fixedpointmath @ git+https://github.com/delvtech/agent_0.git/#subdirectory=lib/fixedpointmath", "numpy", - "requests", "python-dotenv", "web3", # will include eth- packages "hexbytes", diff --git a/lib/chainsync/bin/run_api_server.py b/lib/chainsync/bin/run_api_server.py index c899b9129d..9ed3281b66 100644 --- a/lib/chainsync/bin/run_api_server.py +++ b/lib/chainsync/bin/run_api_server.py @@ -1,7 +1,7 @@ """A simple Flask server to run python scripts.""" import argparse -from chainsync.db import launch_flask +from chainsync.db.api import launch_flask if __name__ == "__main__": parser = argparse.ArgumentParser( diff --git a/lib/chainsync/chainsync/db/__init__.py b/lib/chainsync/chainsync/db/__init__.py index 75cc638cf5..e69de29bb2 100644 --- a/lib/chainsync/chainsync/db/__init__.py +++ b/lib/chainsync/chainsync/db/__init__.py @@ -1,2 +0,0 @@ -"""Api server for the chainsync database.""" -from .api_server import launch_flask diff --git a/lib/chainsync/chainsync/db/api/__init__.py b/lib/chainsync/chainsync/db/api/__init__.py new file mode 100644 index 0000000000..f2b6917e62 --- /dev/null +++ b/lib/chainsync/chainsync/db/api/__init__.py @@ -0,0 +1,3 @@ +"""Api server for the chainsync database.""" +from .api_interface import balance_of, register_username +from .flask_server import launch_flask diff --git a/lib/chainsync/chainsync/db/api/api_interface.py b/lib/chainsync/chainsync/db/api/api_interface.py new file mode 100644 index 0000000000..8c24e624f8 --- /dev/null +++ b/lib/chainsync/chainsync/db/api/api_interface.py @@ -0,0 +1,62 @@ +"""Python api interface for calling the flask server""" +import logging +import time +from http import HTTPStatus + +import pandas as pd +import requests + + +def register_username(api_uri: str, wallet_addrs: list[str], username: str) -> None: + """Registers the username with the flask server. + + Arguments + --------- + register_uri: str + The endpoint for the flask server. + wallet_addrs: list[str] + The list of wallet addresses to register. + username: str + The username to register the wallet addresses under. + """ + # TODO: use the json schema from the server. + json_data = {"wallet_addrs": wallet_addrs, "username": username} + result = requests.post(f"{api_uri}/register_agents", json=json_data, timeout=3) + if result.status_code != HTTPStatus.OK: + raise ConnectionError(result) + + +def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: + """Gets all open positions for a given list of wallet addresses from the db + + Arguments + --------- + api_url : str + The endpoint for the flask server. + wallet_addrs: list[str] + The list of wallet addresses to register. + username: str + The username to register the wallet addresses under. + """ + # TODO: use the json schema from the server. + json_data = {"wallet_addrs": wallet_addrs} + result = None + for _ in range(10): + try: + result = requests.post(f"{api_uri}/balance_of", json=json_data, timeout=3) + break + except requests.exceptions.RequestException: + logging.warning("Connection error to db api server, retrying") + time.sleep(1) + continue + + if result is None or (result.status_code != HTTPStatus.OK): + raise ConnectionError(result) + + # Read json and return + # Since we use pandas write json, we use pandas read json to read, then adjust data + # before returning + # We explicitly set dtype to False to keep everything in string format + # to avoid loss of precision + data = pd.read_json(result.json()["data"], dtype=False) + return data diff --git a/lib/chainsync/chainsync/db/api_server.py b/lib/chainsync/chainsync/db/api/flask_server.py similarity index 100% rename from lib/chainsync/chainsync/db/api_server.py rename to lib/chainsync/chainsync/db/api/flask_server.py diff --git a/lib/chainsync/pyproject.toml b/lib/chainsync/pyproject.toml index e745827b07..4f13645248 100644 --- a/lib/chainsync/pyproject.toml +++ b/lib/chainsync/pyproject.toml @@ -36,6 +36,7 @@ base = [ "psycopg[binary]", "sqlalchemy", "pandas-stubs", + "requests", ] lateral = [ # Lateral dependencies across subpackages are pointing to github From 11693e90197ea1950ff9cf0da9dcf8874a7072b9 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 16:12:22 -0700 Subject: [PATCH 16/31] Renaming back to api_server --- lib/chainsync/chainsync/db/api/__init__.py | 2 +- .../chainsync/db/api/{flask_server.py => api_server.py} | 0 lib/chainsync/chainsync/test_fixtures/db_session.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename lib/chainsync/chainsync/db/api/{flask_server.py => api_server.py} (100%) diff --git a/lib/chainsync/chainsync/db/api/__init__.py b/lib/chainsync/chainsync/db/api/__init__.py index f2b6917e62..96ab512813 100644 --- a/lib/chainsync/chainsync/db/api/__init__.py +++ b/lib/chainsync/chainsync/db/api/__init__.py @@ -1,3 +1,3 @@ """Api server for the chainsync database.""" from .api_interface import balance_of, register_username -from .flask_server import launch_flask +from .api_server import launch_flask diff --git a/lib/chainsync/chainsync/db/api/flask_server.py b/lib/chainsync/chainsync/db/api/api_server.py similarity index 100% rename from lib/chainsync/chainsync/db/api/flask_server.py rename to lib/chainsync/chainsync/db/api/api_server.py diff --git a/lib/chainsync/chainsync/test_fixtures/db_session.py b/lib/chainsync/chainsync/test_fixtures/db_session.py index 5ebf3fc2f6..94239ba1f0 100644 --- a/lib/chainsync/chainsync/test_fixtures/db_session.py +++ b/lib/chainsync/chainsync/test_fixtures/db_session.py @@ -106,7 +106,7 @@ def db_api(psql_docker) -> Iterator[str]: db_api_host = "127.0.0.1" db_api_port = 5005 - api_server_path = Path(__file__).parent.joinpath("../db/api_server") + api_server_path = Path(__file__).parent.joinpath("../db/api/api_server") # Modify an environment to set db credentials env = os.environ.copy() From 18d442a54ee6b890d62d528f8601db019d142267 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Fri, 22 Sep 2023 16:13:42 -0700 Subject: [PATCH 17/31] black --- lib/agent0/agent0/hyperdrive/exec/run_agents.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index 53295500c4..062c30177c 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -14,8 +14,7 @@ from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config from ethpy.base import smart_contract_read -from ethpy.hyperdrive import (HyperdriveAddresses, - fetch_hyperdrive_address_from_uri) +from ethpy.hyperdrive import HyperdriveAddresses, fetch_hyperdrive_address_from_uri from fixedpointmath import FixedPoint from hexbytes import HexBytes from web3.contract.contract import Contract From 8fd2a317e40d08531c2495340c4d2c5caf5e563d Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:00:47 -0700 Subject: [PATCH 18/31] Changing wallet deltas to hyperdrive wallet deltas --- lib/agent0/agent0/base/agents/eth_wallet.py | 4 ++-- .../agent0/hyperdrive/agents/__init__.py | 2 +- .../hyperdrive/agents/hyperdrive_wallet.py | 8 ++++---- .../hyperdrive/exec/execute_agent_trades.py | 18 +++++++++--------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index 0cfb1d6854..bd795fd364 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: # TODO need a base wallet delta - from agent0.hyperdrive.agents import WalletDeltas + from agent0.hyperdrive.agents import HyperdriveWalletDeltas @dataclass(kw_only=True) @@ -44,7 +44,7 @@ def copy(self) -> EthWallet: """Returns a new copy of self""" return EthWallet(**copy.deepcopy(self.__dict__)) - def update(self, wallet_deltas: WalletDeltas) -> None: + def update(self, wallet_deltas: HyperdriveWalletDeltas) -> None: """Update the agent's wallet in-place Arguments diff --git a/lib/agent0/agent0/hyperdrive/agents/__init__.py b/lib/agent0/agent0/hyperdrive/agents/__init__.py index be8d27838b..99325a218c 100644 --- a/lib/agent0/agent0/hyperdrive/agents/__init__.py +++ b/lib/agent0/agent0/hyperdrive/agents/__init__.py @@ -1,3 +1,3 @@ """Account and wallet with Hyperdrive specific parts""" from .hyperdrive_account import HyperdriveAgent -from .hyperdrive_wallet import HyperdriveWallet, Long, Short, WalletDeltas +from .hyperdrive_wallet import HyperdriveWallet, HyperdriveWalletDeltas, Long, Short diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index 1fefa31ab2..20a68807c6 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -13,7 +13,7 @@ @freezable() @dataclass() -class WalletDeltas: +class HyperdriveWalletDeltas: r"""Stores changes for an agent's wallet Arguments @@ -42,9 +42,9 @@ class WalletDeltas: shorts: dict[int, Short] = field(default_factory=dict) withdraw_shares: FixedPoint = FixedPoint(0) - def copy(self) -> WalletDeltas: + def copy(self) -> HyperdriveWalletDeltas: """Returns a new copy of self""" - return WalletDeltas(**copy.deepcopy(self.__dict__)) + return HyperdriveWalletDeltas(**copy.deepcopy(self.__dict__)) @dataclass @@ -166,7 +166,7 @@ def copy(self) -> HyperdriveWallet: """Returns a new copy of self.""" return HyperdriveWallet(**copy.deepcopy(self.__dict__)) - def update(self, wallet_deltas: WalletDeltas) -> None: + def update(self, wallet_deltas: HyperdriveWalletDeltas) -> None: """Update the agent's wallet in-place. Arguments diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index 4079634d98..8b22ba8e4d 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -8,7 +8,7 @@ import eth_utils from agent0.base import Quantity, TokenType -from agent0.hyperdrive.agents import Long, Short, WalletDeltas +from agent0.hyperdrive.agents import HyperdriveWalletDeltas, Long, Short from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction from elfpy import types from elfpy.markets.hyperdrive import HyperdriveMarket @@ -195,7 +195,7 @@ async def async_match_contract_call_to_trade( hyperdrive_contract: Contract, agent: HyperdriveAgent, trade_envelope: types.Trade[HyperdriveMarketAction], -) -> WalletDeltas: +) -> HyperdriveWalletDeltas: """Match statement that executes the smart contract trade based on the provided type. Arguments @@ -251,7 +251,7 @@ async def async_match_contract_call_to_trade( *fn_args, ) maturity_time_seconds = trade_result.maturity_time_seconds - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=-trade_result.base_amount, unit=TokenType.BASE, @@ -286,7 +286,7 @@ async def async_match_contract_call_to_trade( "closeLong", *fn_args, ) - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=trade_result.base_amount, unit=TokenType.BASE, @@ -314,7 +314,7 @@ async def async_match_contract_call_to_trade( *fn_args, ) maturity_time_seconds = trade_result.maturity_time_seconds - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=-trade_result.base_amount, unit=TokenType.BASE, @@ -353,7 +353,7 @@ async def async_match_contract_call_to_trade( "closeShort", *fn_args, ) - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=trade_result.base_amount, unit=TokenType.BASE, @@ -375,7 +375,7 @@ async def async_match_contract_call_to_trade( "addLiquidity", *fn_args, ) - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=-trade_result.base_amount, unit=TokenType.BASE, @@ -393,7 +393,7 @@ async def async_match_contract_call_to_trade( "removeLiquidity", *fn_args, ) - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=trade_result.base_amount, unit=TokenType.BASE, @@ -417,7 +417,7 @@ async def async_match_contract_call_to_trade( "redeemWithdrawalShares", *fn_args, ) - wallet_deltas = WalletDeltas( + wallet_deltas = HyperdriveWalletDeltas( balance=Quantity( amount=trade_result.base_amount, unit=TokenType.BASE, From 91b6f47756b6f761221e04d62619c5cc2fc86d90 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:06:13 -0700 Subject: [PATCH 19/31] Adding wallet deltas base class --- lib/agent0/agent0/base/agents/__init__.py | 2 +- lib/agent0/agent0/base/agents/eth_wallet.py | 28 +++++++++++++++---- .../hyperdrive/agents/hyperdrive_wallet.py | 10 ++----- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/lib/agent0/agent0/base/agents/__init__.py b/lib/agent0/agent0/base/agents/__init__.py index 7d832d0d5b..f7bf2ac2b5 100644 --- a/lib/agent0/agent0/base/agents/__init__.py +++ b/lib/agent0/agent0/base/agents/__init__.py @@ -1,3 +1,3 @@ """Base implementation of agents.""" from .eth_agent import EthAgent -from .eth_wallet import EthWallet +from .eth_wallet import EthWallet, EthWalletDeltas diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index bd795fd364..322a0dbe2d 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -4,16 +4,32 @@ import copy import logging from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import Any -from agent0.base import Quantity, TokenType +from agent0.base import Quantity, TokenType, freezable from elfpy import check_non_zero from fixedpointmath import FixedPoint from hexbytes import HexBytes -if TYPE_CHECKING: - # TODO need a base wallet delta - from agent0.hyperdrive.agents import HyperdriveWalletDeltas + +@dataclass(kw_only=True) +@freezable() +class EthWalletDeltas: + r"""Stores changes for an agent's wallet + + Arguments + ---------- + balance : Quantity + The base assets that held by the trader. + """ + # fungible + balance: Quantity = field(default_factory=lambda: Quantity(amount=FixedPoint(0), unit=TokenType.BASE)) + + # TODO: Support multiple typed balances: + # balance: Dict[TokenType, Quantity] = field(default_factory=dict) + def copy(self) -> EthWalletDeltas: + """Returns a new copy of self""" + return EthWalletDeltas(**copy.deepcopy(self.__dict__)) @dataclass(kw_only=True) @@ -44,7 +60,7 @@ def copy(self) -> EthWallet: """Returns a new copy of self""" return EthWallet(**copy.deepcopy(self.__dict__)) - def update(self, wallet_deltas: HyperdriveWalletDeltas) -> None: + def update(self, wallet_deltas: EthWalletDeltas) -> None: """Update the agent's wallet in-place Arguments diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index 20a68807c6..d38545c5f3 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -6,14 +6,14 @@ from dataclasses import dataclass, field from typing import Iterable -from agent0.base import Quantity, TokenType, freezable -from agent0.base.agents import EthWallet +from agent0.base import freezable +from agent0.base.agents import EthWallet, EthWalletDeltas from fixedpointmath import FixedPoint @freezable() @dataclass() -class HyperdriveWalletDeltas: +class HyperdriveWalletDeltas(EthWalletDeltas): r"""Stores changes for an agent's wallet Arguments @@ -32,10 +32,6 @@ class HyperdriveWalletDeltas: # dataclasses can have many attributes # pylint: disable=too-many-instance-attributes - # fungible - balance: Quantity = field(default_factory=lambda: Quantity(amount=FixedPoint(0), unit=TokenType.BASE)) - # TODO: Support multiple typed balances: - # balance: Dict[TokenType, Quantity] = field(default_factory=dict) lp_tokens: FixedPoint = FixedPoint(0) # non-fungible (identified by key=maturity_time, stored as dict) longs: dict[int, Long] = field(default_factory=dict) From 9d60c6a2257545452f96d324e7a48a9bc37e741f Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:09:06 -0700 Subject: [PATCH 20/31] Fixing docstrings --- .../hyperdrive/agents/hyperdrive_wallet.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py index d38545c5f3..f6d075b4ad 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py @@ -18,16 +18,14 @@ class HyperdriveWalletDeltas(EthWalletDeltas): Arguments ---------- - balance : Quantity - The base assets that held by the trader. lp_tokens : FixedPoint The LP tokens held by the trader. - longs : Dict[FixedPoint, Long] + longs : Dict[int, Long] The long positions held by the trader. - shorts : Dict[FixedPoint, Short] + shorts : Dict[int, Short] The short positions held by the trader. - borrows : Dict[FixedPoint, Borrow] - The borrow positions held by the trader. + withdraw_shares: FixedPoint + The withdraw shares held by the trader. """ # dataclasses can have many attributes # pylint: disable=too-many-instance-attributes @@ -85,10 +83,10 @@ class HyperdriveWallet(EthWallet): The LP tokens held by the trader. withdraw_shares : FixedPoint The amount of unclaimed withdraw shares held by the agent. - longs : Dict[FixedPoint, Long] + longs : Dict[int, Long] The long positions held by the trader. The dictionary is keyed by the maturity time in seconds. - shorts : Dict[FixedPoint, Short] + shorts : Dict[int, Short] The short positions held by the trader. The dictionary is keyed by the maturity time in seconds. """ @@ -105,7 +103,7 @@ def _update_longs(self, longs: Iterable[tuple[int, Long]]) -> None: Arguments --------- - longs : Iterable[tuple[FixedPoint, Long]] + longs : Iterable[tuple[int, Long]] A list (or other Iterable type) of tuples that contain a Long object and its market-relative maturity time """ @@ -134,7 +132,7 @@ def _update_shorts(self, shorts: Iterable[tuple[int, Short]]) -> None: Arguments --------- - shorts : Iterable[tuple[FixedPoint, Short]] + shorts : Iterable[tuple[int, Short]] A list (or other Iterable type) of tuples that contain a Short object and its market-relative mint time """ From 516113db4560edc690d06f04bdb31c1c349aedd5 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:10:06 -0700 Subject: [PATCH 21/31] More docstrings --- lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index 8b22ba8e4d..f860e7a02e 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -204,8 +204,6 @@ async def async_match_contract_call_to_trade( web3 provider object hyperdrive_contract : Contract Any deployed web3 contract - hyperdrive_market : HyperdriveMarket - The elfpy trading market agent : HyperdriveAgent Object containing a wallet address and Elfpy Agent for determining trades trade_object : Trade @@ -213,7 +211,7 @@ async def async_match_contract_call_to_trade( Returns ------- - WalletDeltas + HyperdriveWalletDeltas Deltas to be applied to the agent's wallet """ From 47d50b2bc90ee3996ab020f1a6482f243dc3cdb7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:11:49 -0700 Subject: [PATCH 22/31] Removing unnecessary int casts --- lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index f860e7a02e..12c0fc0df3 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -260,7 +260,7 @@ async def async_match_contract_call_to_trade( case HyperdriveActionType.CLOSE_LONG: if not trade.maturity_time: raise ValueError("Maturity time was not provided, can't close long position.") - maturity_time_seconds = int(trade.maturity_time) + maturity_time_seconds = trade.maturity_time min_output = 0 fn_args = ( maturity_time_seconds, From f0db0c06d56938dc3ed34b5667a3f523d0494846 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:17:39 -0700 Subject: [PATCH 23/31] Removing more unnecessary ints --- lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index 12c0fc0df3..6df56e7a60 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -327,7 +327,7 @@ async def async_match_contract_call_to_trade( case HyperdriveActionType.CLOSE_SHORT: if not trade.maturity_time: raise ValueError("Maturity time was not provided, can't close long position.") - maturity_time_seconds = int(trade.maturity_time) + maturity_time_seconds = trade.maturity_time min_output = 0 fn_args = ( maturity_time_seconds, From f78888ee1481722b2d1671e12fdd1d9d723994ab Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:17:49 -0700 Subject: [PATCH 24/31] docstrings --- .../chainsync/db/api/api_interface.py | 4 +- .../chainsync/test_fixtures/db_session.py | 44 +++++++++++++++---- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/lib/chainsync/chainsync/db/api/api_interface.py b/lib/chainsync/chainsync/db/api/api_interface.py index 8c24e624f8..b188625b6f 100644 --- a/lib/chainsync/chainsync/db/api/api_interface.py +++ b/lib/chainsync/chainsync/db/api/api_interface.py @@ -12,7 +12,7 @@ def register_username(api_uri: str, wallet_addrs: list[str], username: str) -> N Arguments --------- - register_uri: str + api_uri: str The endpoint for the flask server. wallet_addrs: list[str] The list of wallet addresses to register. @@ -35,8 +35,6 @@ def balance_of(api_uri: str, wallet_addrs: list[str]) -> pd.DataFrame: The endpoint for the flask server. wallet_addrs: list[str] The list of wallet addresses to register. - username: str - The username to register the wallet addresses under. """ # TODO: use the json schema from the server. json_data = {"wallet_addrs": wallet_addrs} diff --git a/lib/chainsync/chainsync/test_fixtures/db_session.py b/lib/chainsync/chainsync/test_fixtures/db_session.py index 94239ba1f0..c76fe22e6d 100644 --- a/lib/chainsync/chainsync/test_fixtures/db_session.py +++ b/lib/chainsync/chainsync/test_fixtures/db_session.py @@ -10,6 +10,7 @@ from chainsync import PostgresConfig from chainsync.db.base import Base, initialize_engine from pytest_postgresql.janitor import DatabaseJanitor +from sqlalchemy import Engine from sqlalchemy.orm import Session, sessionmaker # fixture arguments in test function have to be the same as the fixture name @@ -18,7 +19,13 @@ @pytest.fixture(scope="session") def psql_docker() -> Iterator[PostgresConfig]: - """Test fixture for running postgres in docker""" + """Test fixture for running postgres in docker + + Returns + ------- + Iterator[PostgresConfig] + An iterator that yields a PostgresConfig + """ client = docker.from_env() # Using these config for tests @@ -54,8 +61,19 @@ def psql_docker() -> Iterator[PostgresConfig]: @pytest.fixture(scope="session") -def database_engine(psql_docker): - """Test fixture creating psql engine on local postgres container""" +def database_engine(psql_docker: PostgresConfig) -> Iterator[Engine]: + """Test fixture creating psql engine on local postgres container + + Arguments + --------- + psql_docker: PostgresConfig + The PostgresConfig object returned by the `psql_docker` test fixture + + Returns + ------- + Iterator[Engine] + An iterator that yields a sqlalchemy engine + """ # Using default postgres info # Renaming variable to match what it actually is, i.e., the postgres config postgres_config = psql_docker @@ -73,13 +91,18 @@ def database_engine(psql_docker): @pytest.fixture(scope="function") -def db_session(database_engine) -> Iterator[Session]: +def db_session(database_engine: Engine) -> Iterator[Session]: """Initializes the in memory db session and creates the db schema + Arguments + --------- + database_engine : Engine + The sqlalchemy database engine returned from the `database_engine` test fixture + Returns ------- - Iterator[Tuple[Session, str]] - Yields the sqlalchemy session object, with the database api uri + Iterator[Session] + Yields the sqlalchemy session object """ session = sessionmaker(bind=database_engine) @@ -94,13 +117,18 @@ def db_session(database_engine) -> Iterator[Session]: @pytest.fixture(scope="function") -def db_api(psql_docker) -> Iterator[str]: +def db_api(psql_docker: PostgresConfig) -> Iterator[str]: """Launches a process for the db api + Arguments + --------- + psql_docker: PostgresConfig + The PostgresConfig object returned by the `psql_docker` test fixture + Returns ------- Iterator[str] - Yields the sqlalchemy session object, with the database api uri + Yields the database api uri """ # Launch the database api server here db_api_host = "127.0.0.1" From 4c53d66b85d4a6d2f3518954ed89db2a2cf1bbe4 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 11:20:30 -0700 Subject: [PATCH 25/31] Fixing issue with order of decorators --- lib/agent0/agent0/base/agents/eth_wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/agents/eth_wallet.py index 322a0dbe2d..8ab3129703 100644 --- a/lib/agent0/agent0/base/agents/eth_wallet.py +++ b/lib/agent0/agent0/base/agents/eth_wallet.py @@ -12,8 +12,8 @@ from hexbytes import HexBytes -@dataclass(kw_only=True) @freezable() +@dataclass() class EthWalletDeltas: r"""Stores changes for an agent's wallet From 2b766d0edcc57d74a943b15fbb739fa048ff9415 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:15:21 -0700 Subject: [PATCH 26/31] Using agent0 action types and market instead of elfpy, fixing fixedpoint for maturity time --- .../hyperdrive/agents/hyperdrive_account.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py index a70c1252ad..e58838a9dc 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py @@ -7,10 +7,10 @@ from agent0.base import Quantity, TokenType from agent0.base.agents import EthAgent from agent0.base.policies import BasePolicy -from elfpy.markets.hyperdrive import HyperdriveMarket, HyperdriveMarketAction, MarketActionType +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from elfpy.markets.hyperdrive import HyperdriveMarket from elfpy.types import MarketType, Trade from eth_account.signers.local import LocalAccount -from fixedpointmath import FixedPoint from hexbytes import HexBytes from .hyperdrive_wallet import HyperdriveWallet @@ -91,11 +91,10 @@ def liquidation_trades(self) -> list[Trade[MarketAction]]: Trade( market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( - action_type=MarketActionType.CLOSE_LONG, + action_type=HyperdriveActionType.CLOSE_LONG, trade_amount=long.balance, wallet=self.wallet, # type: ignore - # TODO this should be in int - maturity_time=FixedPoint(maturity_time), + maturity_time=maturity_time, ), ) ) @@ -107,10 +106,10 @@ def liquidation_trades(self) -> list[Trade[MarketAction]]: Trade( market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( - action_type=MarketActionType.CLOSE_SHORT, + action_type=HyperdriveActionType.CLOSE_SHORT, trade_amount=short.balance, wallet=self.wallet, # type: ignore - maturity_time=FixedPoint(maturity_time), + maturity_time=maturity_time, ), ) ) @@ -121,7 +120,7 @@ def liquidation_trades(self) -> list[Trade[MarketAction]]: Trade( market_type=MarketType.HYPERDRIVE, market_action=HyperdriveMarketAction( - action_type=MarketActionType.REMOVE_LIQUIDITY, + action_type=HyperdriveActionType.REMOVE_LIQUIDITY, trade_amount=self.wallet.lp_tokens, wallet=self.wallet, # type: ignore ), From 6cfc6cac176d762a58a8bfe671d653db171d06ed Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:29:58 -0700 Subject: [PATCH 27/31] Moving wallet info from agent to state --- lib/agent0/agent0/base/agents/__init__.py | 1 - lib/agent0/agent0/base/agents/eth_agent.py | 3 +-- lib/agent0/agent0/base/state/__init__.py | 1 + lib/agent0/agent0/base/{agents => state}/eth_wallet.py | 0 lib/agent0/agent0/hyperdrive/agents/__init__.py | 1 - lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py | 4 +--- lib/agent0/agent0/hyperdrive/state/__init__.py | 1 + lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py | 3 ++- .../agent0/hyperdrive/{agents => state}/hyperdrive_wallet.py | 0 9 files changed, 6 insertions(+), 8 deletions(-) rename lib/agent0/agent0/base/{agents => state}/eth_wallet.py (100%) rename lib/agent0/agent0/hyperdrive/{agents => state}/hyperdrive_wallet.py (100%) diff --git a/lib/agent0/agent0/base/agents/__init__.py b/lib/agent0/agent0/base/agents/__init__.py index f7bf2ac2b5..cf34045ba6 100644 --- a/lib/agent0/agent0/base/agents/__init__.py +++ b/lib/agent0/agent0/base/agents/__init__.py @@ -1,3 +1,2 @@ """Base implementation of agents.""" from .eth_agent import EthAgent -from .eth_wallet import EthWallet, EthWalletDeltas diff --git a/lib/agent0/agent0/base/agents/eth_agent.py b/lib/agent0/agent0/base/agents/eth_agent.py index 84d74db507..0f6eb9624f 100644 --- a/lib/agent0/agent0/base/agents/eth_agent.py +++ b/lib/agent0/agent0/base/agents/eth_agent.py @@ -5,14 +5,13 @@ from agent0.base import Quantity, TokenType from agent0.base.policies import BasePolicy, NoActionPolicy +from agent0.base.state import EthWallet from elfpy.types import Trade from eth_account.signers.local import LocalAccount from eth_typing import ChecksumAddress from hexbytes import HexBytes from web3 import Web3 -from .eth_wallet import EthWallet - Policy = TypeVar("Policy", bound=BasePolicy) Market = TypeVar("Market") MarketAction = TypeVar("MarketAction") diff --git a/lib/agent0/agent0/base/state/__init__.py b/lib/agent0/agent0/base/state/__init__.py index 8b7f78f560..9879931e29 100644 --- a/lib/agent0/agent0/base/state/__init__.py +++ b/lib/agent0/agent0/base/state/__init__.py @@ -1,2 +1,3 @@ """Base classes for interface objects""" +from .eth_wallet import EthWallet, EthWalletDeltas from .market_state import BaseMarketState diff --git a/lib/agent0/agent0/base/agents/eth_wallet.py b/lib/agent0/agent0/base/state/eth_wallet.py similarity index 100% rename from lib/agent0/agent0/base/agents/eth_wallet.py rename to lib/agent0/agent0/base/state/eth_wallet.py diff --git a/lib/agent0/agent0/hyperdrive/agents/__init__.py b/lib/agent0/agent0/hyperdrive/agents/__init__.py index 99325a218c..b7961dd374 100644 --- a/lib/agent0/agent0/hyperdrive/agents/__init__.py +++ b/lib/agent0/agent0/hyperdrive/agents/__init__.py @@ -1,3 +1,2 @@ """Account and wallet with Hyperdrive specific parts""" from .hyperdrive_account import HyperdriveAgent -from .hyperdrive_wallet import HyperdriveWallet, HyperdriveWalletDeltas, Long, Short diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py index e58838a9dc..edcb08ba24 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py @@ -7,14 +7,12 @@ from agent0.base import Quantity, TokenType from agent0.base.agents import EthAgent from agent0.base.policies import BasePolicy -from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction, HyperdriveWallet from elfpy.markets.hyperdrive import HyperdriveMarket from elfpy.types import MarketType, Trade from eth_account.signers.local import LocalAccount from hexbytes import HexBytes -from .hyperdrive_wallet import HyperdriveWallet - Policy = TypeVar("Policy", bound=BasePolicy) Market = TypeVar( "Market", bound=HyperdriveMarket diff --git a/lib/agent0/agent0/hyperdrive/state/__init__.py b/lib/agent0/agent0/hyperdrive/state/__init__.py index 42835cffbe..11802723b6 100644 --- a/lib/agent0/agent0/hyperdrive/state/__init__.py +++ b/lib/agent0/agent0/hyperdrive/state/__init__.py @@ -1,4 +1,5 @@ """Stateful objects for Hyperdrive AMM""" from .hyperdrive_actions import HyperdriveActionType, HyperdriveMarketAction from .hyperdrive_market_state import HyperdriveMarketState +from .hyperdrive_wallet import HyperdriveWallet, HyperdriveWalletDeltas, Long, Short from .trade_result import HyperdriveActionResult diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py index 0bfb24f12b..410567fe9c 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_actions.py @@ -5,10 +5,11 @@ from enum import Enum from agent0.base import freezable -from agent0.hyperdrive.agents import HyperdriveWallet from elfpy.markets.base import BaseMarketAction from fixedpointmath import FixedPoint +from .hyperdrive_wallet import HyperdriveWallet + class HyperdriveActionType(Enum): r"""The descriptor of an action in a market""" diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py similarity index 100% rename from lib/agent0/agent0/hyperdrive/agents/hyperdrive_wallet.py rename to lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py From ad5fa1102522833bcb40dbefb352158dbcadc6d7 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:30:59 -0700 Subject: [PATCH 28/31] Fixing import --- lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py b/lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py index f6d075b4ad..ed0d997755 100644 --- a/lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py +++ b/lib/agent0/agent0/hyperdrive/state/hyperdrive_wallet.py @@ -7,7 +7,7 @@ from typing import Iterable from agent0.base import freezable -from agent0.base.agents import EthWallet, EthWalletDeltas +from agent0.base.state import EthWallet, EthWalletDeltas from fixedpointmath import FixedPoint From 70544ca7def80ed065f6dd53cc4567705c557aee Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:32:00 -0700 Subject: [PATCH 29/31] Casting fixed point from market to ints for maturity_time --- lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py index edcb08ba24..f87361afc5 100644 --- a/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py +++ b/lib/agent0/agent0/hyperdrive/agents/hyperdrive_account.py @@ -145,7 +145,10 @@ def get_trades(self, market: Market) -> list[Trade[MarketAction]]: # edit each action in place for action in actions: if action.market_type == MarketType.HYPERDRIVE and action.market_action.maturity_time is None: - action.market_action.maturity_time = market.latest_checkpoint_time + market.position_duration.seconds + # TODO market latest_checkpoint_time and position_duration should be in ints + action.market_action.maturity_time = int(market.latest_checkpoint_time) + int( + market.position_duration.seconds + ) if action.market_action.trade_amount <= 0: raise ValueError("Trade amount cannot be zero or negative.") return actions From f4105b9b187f9cba8021f9d0bb62933c2643da98 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:35:51 -0700 Subject: [PATCH 30/31] Fixing imports for wallet states --- lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py | 3 +-- lib/agent0/agent0/hyperdrive/exec/run_agents.py | 2 +- lib/agent0/agent0/hyperdrive/policies/hyperdrive_policy.py | 3 +-- lib/agent0/agent0/hyperdrive/policies/random_agent.py | 2 +- lib/agent0/agent0/hyperdrive/policies/smart_long.py | 2 +- lib/agent0/agent0/hyperdrive/policies/smart_short.py | 2 +- lib/agent0/agent0/test_fixtures/cycle_trade_policy.py | 3 +-- lib/agent0/examples/example_agent.py | 2 +- lib/agent0/examples/hyperdrive_agents.py | 2 +- tests/bot_load_state_test.py | 3 +-- 10 files changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py index 6df56e7a60..ee06c24bf3 100644 --- a/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py +++ b/lib/agent0/agent0/hyperdrive/exec/execute_agent_trades.py @@ -8,8 +8,7 @@ import eth_utils from agent0.base import Quantity, TokenType -from agent0.hyperdrive.agents import HyperdriveWalletDeltas, Long, Short -from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction, HyperdriveWalletDeltas, Long, Short from elfpy import types from elfpy.markets.hyperdrive import HyperdriveMarket from ethpy.base import ( diff --git a/lib/agent0/agent0/hyperdrive/exec/run_agents.py b/lib/agent0/agent0/hyperdrive/exec/run_agents.py index 062c30177c..13a7d34abc 100644 --- a/lib/agent0/agent0/hyperdrive/exec/run_agents.py +++ b/lib/agent0/agent0/hyperdrive/exec/run_agents.py @@ -9,7 +9,7 @@ from agent0 import AccountKeyConfig from agent0.base import Quantity, TokenType from agent0.base.config import DEFAULT_USERNAME, AgentConfig, EnvironmentConfig -from agent0.hyperdrive.agents import HyperdriveWallet, Long, Short +from agent0.hyperdrive.state import HyperdriveWallet, Long, Short from chainsync.db.api import balance_of, register_username from eth_typing import BlockNumber from ethpy import EthConfig, build_eth_config diff --git a/lib/agent0/agent0/hyperdrive/policies/hyperdrive_policy.py b/lib/agent0/agent0/hyperdrive/policies/hyperdrive_policy.py index b0e84c546d..4b788ca4e0 100644 --- a/lib/agent0/agent0/hyperdrive/policies/hyperdrive_policy.py +++ b/lib/agent0/agent0/hyperdrive/policies/hyperdrive_policy.py @@ -2,8 +2,7 @@ # from agent0.hyperdrive import HyperdriveMarketState # TODO: use agent0 market state instead of elfpy market from agent0.base.policies import BasePolicy -from agent0.hyperdrive.agents import HyperdriveWallet -from agent0.hyperdrive.state import HyperdriveMarketAction +from agent0.hyperdrive.state import HyperdriveMarketAction, HyperdriveWallet from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState from elfpy.types import Trade diff --git a/lib/agent0/agent0/hyperdrive/policies/random_agent.py b/lib/agent0/agent0/hyperdrive/policies/random_agent.py index 263b052482..9e7122584e 100644 --- a/lib/agent0/agent0/hyperdrive/policies/random_agent.py +++ b/lib/agent0/agent0/hyperdrive/policies/random_agent.py @@ -11,7 +11,7 @@ from .hyperdrive_policy import HyperdrivePolicy if TYPE_CHECKING: - from agent0.hyperdrive.agents import HyperdriveWallet + from agent0.hyperdrive.state import HyperdriveWallet # from agent0.hyperdrive import HyperdriveMarketState # TODO: use agent0 market state instead of elfpy market from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState diff --git a/lib/agent0/agent0/hyperdrive/policies/smart_long.py b/lib/agent0/agent0/hyperdrive/policies/smart_long.py index 858a524c58..291143ea98 100644 --- a/lib/agent0/agent0/hyperdrive/policies/smart_long.py +++ b/lib/agent0/agent0/hyperdrive/policies/smart_long.py @@ -11,7 +11,7 @@ from .hyperdrive_policy import HyperdrivePolicy if TYPE_CHECKING: - from agent0.hyperdrive.agents import HyperdriveWallet + from agent0.hyperdrive.state import HyperdriveWallet # from agent0.hyperdrive import HyperdriveMarketState # TODO: use agent0 market state instead of elfpy market from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState diff --git a/lib/agent0/agent0/hyperdrive/policies/smart_short.py b/lib/agent0/agent0/hyperdrive/policies/smart_short.py index 9e8cb45157..8c2ad9dfac 100644 --- a/lib/agent0/agent0/hyperdrive/policies/smart_short.py +++ b/lib/agent0/agent0/hyperdrive/policies/smart_short.py @@ -11,7 +11,7 @@ from .hyperdrive_policy import HyperdrivePolicy if TYPE_CHECKING: - from agent0.hyperdrive.agents import HyperdriveWallet + from agent0.hyperdrive.state import HyperdriveWallet # from agent0.hyperdrive import HyperdriveMarketState # TODO: use agent0 market state instead of elfpy market from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState diff --git a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py index 31459fdc6a..de08bbe006 100644 --- a/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py +++ b/lib/agent0/agent0/test_fixtures/cycle_trade_policy.py @@ -4,9 +4,8 @@ from typing import Type import pytest -from agent0.hyperdrive.agents import HyperdriveWallet from agent0.hyperdrive.policies import HyperdrivePolicy -from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction, HyperdriveWallet from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState from elfpy.types import MarketType, Trade from fixedpointmath import FixedPoint diff --git a/lib/agent0/examples/example_agent.py b/lib/agent0/examples/example_agent.py index 0af34e19cc..960eb9cf6e 100644 --- a/lib/agent0/examples/example_agent.py +++ b/lib/agent0/examples/example_agent.py @@ -13,7 +13,7 @@ from fixedpointmath import FixedPoint if TYPE_CHECKING: - from agent0.hyperdrive.agents import HyperdriveWallet + from agent0.hyperdrive.state import HyperdriveWallet from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState from numpy.random._generator import Generator as NumpyGenerator diff --git a/lib/agent0/examples/hyperdrive_agents.py b/lib/agent0/examples/hyperdrive_agents.py index a2e6c64b64..646fe68bfb 100644 --- a/lib/agent0/examples/hyperdrive_agents.py +++ b/lib/agent0/examples/hyperdrive_agents.py @@ -11,7 +11,7 @@ from fixedpointmath import FixedPoint if TYPE_CHECKING: - from agent0.hyperdrive.agents import HyperdriveWallet + from agent0.hyperdrive.state import HyperdriveWallet from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState from numpy.random._generator import Generator as NumpyGenerator diff --git a/tests/bot_load_state_test.py b/tests/bot_load_state_test.py index 2c6accde3a..5c2f166129 100644 --- a/tests/bot_load_state_test.py +++ b/tests/bot_load_state_test.py @@ -6,10 +6,9 @@ from agent0 import build_account_key_config_from_agent_config from agent0.base.config import AgentConfig, EnvironmentConfig -from agent0.hyperdrive.agents import HyperdriveWallet from agent0.hyperdrive.exec import run_agents from agent0.hyperdrive.policies import HyperdrivePolicy -from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction +from agent0.hyperdrive.state import HyperdriveActionType, HyperdriveMarketAction, HyperdriveWallet from agent0.test_fixtures import AgentDoneException from chainsync.exec import acquire_data, data_analysis from elfpy.markets.hyperdrive import HyperdriveMarket as HyperdriveMarketState From aeee11c1a87becae8f17f4721ef76ae7dadfb864 Mon Sep 17 00:00:00 2001 From: Sheng Lundquist Date: Mon, 25 Sep 2023 12:42:32 -0700 Subject: [PATCH 31/31] One more import fix --- lib/agent0/agent0/base/policies/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/agent0/agent0/base/policies/base.py b/lib/agent0/agent0/base/policies/base.py index d07854b99c..540d3b32b9 100644 --- a/lib/agent0/agent0/base/policies/base.py +++ b/lib/agent0/agent0/base/policies/base.py @@ -8,7 +8,7 @@ from numpy.random import default_rng if TYPE_CHECKING: - from agent0.base.agents import EthWallet + from agent0.base.state import EthWallet # from agent0.base.state import BaseMarketState # TODO: don't rely on elfpy base market from elfpy.markets.base import BaseMarket as BaseMarketState