Skip to content

Using decimal instead of floats for postgres ingestion #782

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slundqui merged 6 commits into from
Aug 3, 2023
Merged

Using decimal instead of floats for postgres ingestion #782

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d39f7bc
Using decimal instead of floats for postgres ingestion
Aug 3, 2023
091f1df
Updating hyperdrive package to v0.0.11
Aug 3, 2023
97df976
Adding lpSharePrice to poolinfo. Fixing issue with pool config equali…
Aug 3, 2023
fd00a17
Unused import
Aug 3, 2023
6d03616
Setting decimal constants as strings
Aug 3, 2023
16bcfe3
Removing dead comments
Aug 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,903 changes: 970 additions & 933 deletions packages/hyperdrive/src/abis/IHyperdrive.sol/IHyperdrive.json
View file
Open in desktop

Large diffs are not rendered by default.

32 changes: 15 additions & 17 deletions src/data/convert_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import logging
import time
from decimal import Decimal
from typing import Any

from eth_typing import BlockNumber
Expand Down Expand Up @@ -222,7 +223,7 @@ def convert_pool_config(pool_config_dict: dict[str, Any]) -> db_schema.PoolConfi
else:
value = pool_config_dict[key]
if isinstance(value, FixedPoint):
value = float(value)
value = Decimal(str(value))
args_dict[key] = value
pool_config = db_schema.PoolConfig(**args_dict)
return pool_config
Expand All @@ -249,7 +250,7 @@ def convert_pool_info(pool_info_dict: dict[str, Any]) -> db_schema.PoolInfo:
else:
value = pool_info_dict[key]
if isinstance(value, FixedPoint):
value = float(value)
value = Decimal(str(value))
args_dict[key] = value
block_pool_info = db_schema.PoolInfo(**args_dict)
return block_pool_info
Expand Down Expand Up @@ -277,34 +278,31 @@ def convert_checkpoint_info(checkpoint_info_dict: dict[str, Any]) -> db_schema.C
else:
value = checkpoint_info_dict[key]
if isinstance(value, FixedPoint):
value = float(value)
value = Decimal(str(value))
args_dict[key] = value
block_checkpoint_info = db_schema.CheckpointInfo(**args_dict)
return block_checkpoint_info


def _convert_scaled_value(input_val: int | None) -> float | None:
def _convert_scaled_value(input_val: int | None) -> Decimal | None:
"""
Given a scaled value int, converts it to a float, while supporting Nones
Given a scaled value int, converts it to a Decimal, while supporting Nones

Arguments
----------
input_val: int | None
The scaled integer value to unscale and convert to float
The scaled integer value to unscale and convert to Decimal

Returns
-------
float | None
The unscaled floating point value

Note
----
We cast to FixedPoint, then to floats to keep noise to a minimum.
There is no loss of precision when going from Fixedpoint to float.
Once this is fed into postgres, postgres will use the fixed-precision Numeric type.
Decimal | None
The unscaled Decimal value
"""
if input_val is not None:
return float(FixedPoint(scaled_value=input_val))
# TODO add this cast within fixedpoint
fp_val = FixedPoint(scaled_value=input_val)
str_val = str(fp_val)
return Decimal(str_val)
return None


Expand Down Expand Up @@ -638,7 +636,7 @@ def _recursive_dict_conversion(obj: Any) -> Any:

def _query_contract_for_balance(
contract: Contract, wallet_addr: str, block_number: BlockNumber, token_id: int | None = None
) -> float | None:
) -> Decimal | None:
"""Queries the given contract for the wallet's token_id balance.

Arguments
Expand All @@ -654,7 +652,7 @@ def _query_contract_for_balance(

Returns
-------
float | None
Decimal | None
The amount token_id in wallet_addr. None if failed
"""
num_token_scaled = None
Expand Down
80 changes: 41 additions & 39 deletions src/data/db_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from __future__ import annotations

from datetime import datetime
from decimal import Decimal
from typing import Union

from sqlalchemy import BigInteger, Boolean, DateTime, ForeignKey, Integer, Numeric, String
Expand Down Expand Up @@ -41,9 +42,9 @@ class WalletInfo(Base):
baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
# tokenType is the baseTokenType appended with "-<maturity_time>" for LONG and SHORT
tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None)
tokenValue: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
maturityTime: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
sharePrice: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
tokenValue: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
maturityTime: Mapped[Union[int, None]] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), default=None)
sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)


class WalletDelta(Base):
Expand All @@ -64,8 +65,8 @@ class WalletDelta(Base):
baseTokenType: Mapped[Union[str, None]] = mapped_column(String, index=True, default=None)
# tokenType is the baseTokenType appended with "-<maturity_time>" for LONG and SHORT
tokenType: Mapped[Union[str, None]] = mapped_column(String, default=None)
delta: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
maturityTime: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
delta: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
maturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)


class PoolConfig(Base):
Expand All @@ -75,20 +76,20 @@ class PoolConfig(Base):

contractAddress: Mapped[str] = mapped_column(String, primary_key=True)
baseToken: Mapped[Union[str, None]] = mapped_column(String, default=None)
initialSharePrice: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
minimumShareReserves: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
initialSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
minimumShareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
positionDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
checkpointDuration: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
timeStretch: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
timeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
governance: Mapped[Union[str, None]] = mapped_column(String, default=None)
feeCollector: Mapped[Union[str, None]] = mapped_column(String, default=None)
curveFee: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
flatFee: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
governanceFee: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
oracleSize: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
curveFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
flatFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
governanceFee: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
oracleSize: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
updateGap: Mapped[Union[int, None]] = mapped_column(Integer, default=None)
invTimeStretch: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
termLength: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
invTimeStretch: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
termLength: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)


class CheckpointInfo(Base):
Expand All @@ -98,9 +99,9 @@ class CheckpointInfo(Base):

blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True)
timestamp: Mapped[datetime] = mapped_column(DateTime, index=True)
sharePrice: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
longSharePrice: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
shortBaseVolume: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
longSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)


class PoolInfo(Base):
Expand All @@ -113,18 +114,19 @@ class PoolInfo(Base):

blockNumber: Mapped[int] = mapped_column(BigInteger, primary_key=True)
timestamp: Mapped[datetime] = mapped_column(DateTime, index=True)
shareReserves: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
bondReserves: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
lpTotalSupply: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
sharePrice: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
longsOutstanding: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
longAverageMaturityTime: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
shortsOutstanding: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
shortAverageMaturityTime: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
shortBaseVolume: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
withdrawalSharesReadyToWithdraw: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
withdrawalSharesProceeds: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
totalSupplyWithdrawalShares: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
shareReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
bondReserves: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
lpTotalSupply: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
sharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
lpSharePrice: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
longsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
longAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
shortsOutstanding: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
shortAverageMaturityTime: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
shortBaseVolume: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
withdrawalSharesReadyToWithdraw: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
withdrawalSharesProceeds: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
totalSupplyWithdrawalShares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)


class Transaction(Base):
Expand Down Expand Up @@ -159,20 +161,20 @@ class Transaction(Base):
input_method: Mapped[Union[str, None]] = mapped_column(String, default=None)

# Method: initialize
input_params_contribution: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_apr: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_contribution: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
input_params_apr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
input_params_destination: Mapped[Union[str, None]] = mapped_column(String, default=None)
input_params_asUnderlying: Mapped[Union[bool, None]] = mapped_column(Boolean, default=None)

# Method: openLong
input_params_baseAmount: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_minOutput: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_baseAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
input_params_minOutput: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
# input_params_destination
# input_params_asUnderlying

# Method: openShort
input_params_bondAmount: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_maxDeposit: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_bondAmount: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
input_params_maxDeposit: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
# input_params_destination
# input_params_asUnderlying

Expand All @@ -192,13 +194,13 @@ class Transaction(Base):

# Method: addLiquidity
# input_params_contribution
input_params_minApr: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_maxApr: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_minApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
input_params_maxApr: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
# input_params_destination
# input_params_asUnderlying

# Method: removeLiquidity
input_params_shares: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
input_params_shares: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
# input_params_minOutput
# input_params_destination
# input_params_asUnderlying
Expand All @@ -210,7 +212,7 @@ class Transaction(Base):
# args_owner
# args_spender
# args_id
event_value: Mapped[Union[float, None]] = mapped_column(Numeric, default=None)
event_value: Mapped[Union[Decimal, None]] = mapped_column(Numeric, default=None)
event_operator: Mapped[Union[str, None]] = mapped_column(String, default=None)
event_id: Mapped[Union[int, None]] = mapped_column(Numeric, default=None)
# Fields calculated from base
Expand Down
8 changes: 5 additions & 3 deletions src/data/postgres.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,9 @@ def add_pool_config(pool_config: PoolConfig, session: Session) -> None:
# if multiple threads try to add pool config at the same time
# This function is being called by acquire_data.py, which should only have one
# instance per db, so no need to worry about it here
existing_pool_config = get_pool_config(session, contract_address=pool_config.contractAddress)

# Since we're doing a direct equality comparison, we don't want to coerce into floats here
existing_pool_config = get_pool_config(session, contract_address=pool_config.contractAddress, coerce_float=False)

if len(existing_pool_config) == 0:
session.add(pool_config)
Expand Down Expand Up @@ -340,7 +342,7 @@ def add_user_map(username: str, addresses: list[str], session: Session) -> None:
raise err


def get_pool_config(session: Session, contract_address: str | None = None) -> pd.DataFrame:
def get_pool_config(session: Session, contract_address: str | None = None, coerce_float=True) -> pd.DataFrame:
"""Get all pool config and returns as a pandas dataframe.

Arguments
Expand All @@ -358,7 +360,7 @@ def get_pool_config(session: Session, contract_address: str | None = None) -> pd
query = session.query(PoolConfig)
if contract_address is not None:
query = query.filter(PoolConfig.contractAddress == contract_address)
return pd.read_sql(query.statement, con=session.connection())
return pd.read_sql(query.statement, con=session.connection(), coerce_float=coerce_float)


def get_pool_info(session: Session, start_block: int | None = None, end_block: int | None = None) -> pd.DataFrame:
Expand Down
9 changes: 5 additions & 4 deletions src/data/tests/test_checkpoint.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""CRUD tests for CheckpointInfo"""
from datetime import datetime
from decimal import Decimal

import numpy as np
import pytest
Expand Down Expand Up @@ -50,7 +51,7 @@ def test_update_checkpoint(self, session):
postgres.add_checkpoint_infos([checkpoint], session)
session.commit()

checkpoint.sharePrice = 5.0
checkpoint.sharePrice = Decimal("5.0")
session.commit()

updated_checkpoint = session.query(CheckpointInfo).filter_by(blockNumber=1).first()
Expand Down Expand Up @@ -106,9 +107,9 @@ def test_get_checkpoints(self, session):

def test_block_query_checkpoints(self, session):
"""Testing querying by block number of checkpoints via interface"""
checkpoint_1 = CheckpointInfo(blockNumber=0, timestamp=datetime.now(), sharePrice=3.1)
checkpoint_2 = CheckpointInfo(blockNumber=1, timestamp=datetime.now(), sharePrice=3.2)
checkpoint_3 = CheckpointInfo(blockNumber=2, timestamp=datetime.now(), sharePrice=3.3)
checkpoint_1 = CheckpointInfo(blockNumber=0, timestamp=datetime.now(), sharePrice=Decimal("3.1"))
checkpoint_2 = CheckpointInfo(blockNumber=1, timestamp=datetime.now(), sharePrice=Decimal("3.2"))
checkpoint_3 = CheckpointInfo(blockNumber=2, timestamp=datetime.now(), sharePrice=Decimal("3.3"))
postgres.add_checkpoint_infos([checkpoint_1, checkpoint_2, checkpoint_3], session)

checkpoints_df = postgres.get_checkpoint_info(session, start_block=1)
Expand Down
Loading