ruff-ed with flake8-bugbear (#221)

* ruff-ed with flake8-bugbear

* add docs

HISTORY.rst

@@ -2,6 +2,9 @@
 
 History
 -------
+0.8.105 [2023-05-16]
+* chore: ruff-ed
+
 0.8.104 [2023-05-15]
 * fix: move get_dt() and get_block() to BlockNumber object
 * feat: enrich command-line for create, help and test-all

credmark/cmf/engine/context.py

@@ -33,7 +33,7 @@
     BlockNumberOutOfRangeDetailDTO,
     BlockNumberOutOfRangeError,
 )
-from credmark.dto import DTOType, DTOValidationError, EmptyInput
+from credmark.dto import DTOType, DTOValidationError
 from credmark.dto.encoder import json_dumps
 from credmark.dto.transform import (
     DataTransformError,
@@ -434,7 +434,7 @@ def _use_no_local_model(self):
 
     def run_model(self,
                   slug,
-                  input=EmptyInput(),
+                  input,
                   return_type=None,
                   block_number: Union[BlockNumber, int, None] = None,
                   version=None,
@@ -760,7 +760,7 @@ def _run_local_model_with_class(self,  # pylint: disable=too-many-locals,too-man
             except DataTransformError as err:
                 # We convert to an input error here to distinguish
                 # from output transform errors below
-                raise ModelInputError(str(err))
+                raise ModelInputError(str(err)) from None
 
             input_as_dict = transform_dto_to_dict(input)
 
@@ -807,7 +807,7 @@ def _run_local_model_with_class(self,  # pylint: disable=too-many-locals,too-man
                         output = json.loads(json_dumps(output))
 
                 except DataTransformError as err:
-                    raise ModelOutputError(str(err))
+                    raise ModelOutputError(str(err)) from None
 
             EngineModelContext.notify_model_run(slug,
                                                 model_class.version,

credmark/cmf/engine/mocks.py

@@ -313,7 +313,7 @@ def _load_module_symbol(self, modulename, name):
             value = vars(module)[name]
         except (ImportError, KeyError) as err:
             raise Exception(
-                f'Error importing symbol {name} from module {modulename}: {err}')
+                f'Error importing symbol {name} from module {modulename}: {err}') from None
         return value
 
     def _load_configuration(self, config_module_name: str):
@@ -324,7 +324,7 @@ def _load_configuration(self, config_module_name: str):
                 self.add_mock_configuration(config)
         except Exception as exc:
             raise Exception(
-                f'Error loading mocks mock module {config_module_name}: {exc}')
+                f'Error loading mocks mock module {config_module_name}: {exc}') from None
 
     def _split_output_match_list(self, slug: str, mocks: List[ModelMock]):
         output_list = []

credmark/cmf/engine/model_api.py

@@ -172,7 +172,7 @@ def run_model(self,  # pylint: disable=too-many-arguments,too-many-locals,too-ma
             logger.error(
                 f'Error running api request for {slug} {self.__url}: {err}')
             raise ModelRunRequestError(
-                f'Unable to reach model runner for model {slug}', '503')
+                f'Unable to reach model runner for model {slug}', '503') from None
         except ModelBaseError:
             raise
         except Exception as err:
@@ -185,15 +185,15 @@ def run_model(self,  # pylint: disable=too-many-arguments,too-many-locals,too-ma
                     error_result = resp.json()
                 except Exception:
                     raise ModelRunRequestError(
-                        f'Model run error response: {resp.text}', str(resp.status_code))
+                        f'Model run error response: {resp.text}', str(resp.status_code)) from None
 
                 raise ModelRunRequestError(
                     str(error_result.get('message', 'Error response from runner api')),
                     str(error_result.get('statusCode', 500)),
-                )
+                ) from None
 
             raise ModelRunRequestError(
-                f'Model run request error: {str(err)}', str(503))
+                f'Model run request error: {str(err)}', str(503)) from None
         finally:
             # Ensure the response is closed in case we ever don't
             # read the content.

credmark/cmf/engine/model_loader.py

@@ -189,7 +189,7 @@ def _load_json_file(self, path):
             try:
                 return json.load(stream)
             except Exception as exc:
-                raise Exception(f'Error loading json manifest: {exc}')
+                raise Exception(f'Error loading json manifest: {exc}') from None
 
     def _process_model_manifest(self, manifest, base_path, fpath):
         models: list = manifest.get('models')
@@ -228,7 +228,7 @@ def _process_model_manifest_entry(self, model_manifest, mclass):
             model_manifest['version'] = str(model_manifest['version'])
         except KeyError as err:
             raise Exception(
-                f'Missing field {err} for model {model_manifest.get("slug", "<unknown>")}')
+                f'Missing field {err} for model {model_manifest.get("slug", "<unknown>")}') from None
 
         if mclass is not None:
             self._add_model_class(mclass)
@@ -270,7 +270,7 @@ def _add_model_class(self, model_class: Type[Model]):
         try:
             pver = version.Version(ver)
         except version.InvalidVersion:
-            raise Exception(f'${model_class} has invalid version ({ver})')
+            raise Exception(f'${model_class} has invalid version ({ver})') from None
 
         self._add_slug_version_model_class(slug, pver, model_class)
 

credmark/cmf/engine/web3_registry.py

@@ -50,7 +50,7 @@ def load_providers_from_env():
                 chain_to_provider_url: dict = json.loads(providers_json)
             except Exception as err:
                 raise Exception(
-                    f'Error parsing JSON in env var CREDMARK_WEB3_PROVIDERS: {err}')
+                    f'Error parsing JSON in env var CREDMARK_WEB3_PROVIDERS: {err}') from None
         else:
             chain_to_provider_url = {}
 

credmark/cmf/model/__init__.py

@@ -208,10 +208,10 @@ def get_attr(self, attr_name=attr_name):
         setattr(cls, attr_name, attr_prop)
         setattr(cls, '_' + attr_name, attr_value)
 
-        setattr(cls, 'slug', slug)
-        setattr(cls, 'version', version)
-        setattr(cls, 'inputDTO', input)
-        setattr(cls, 'outputDTO', output)
+        cls.slug = slug
+        cls.version = version
+        cls.inputDTO = input
+        cls.outputDTO = output
 
         # Checks for the class
         # 1. it need to be inherited from Model class

credmark/cmf/model/context.py

@@ -8,7 +8,7 @@
 
 from credmark.cmf.engine.web3_registry import Web3Registry
 from credmark.cmf.types import BlockNumber, Network
-from credmark.dto import DTOType, EmptyInput
+from credmark.dto import DTOType
 
 from .errors import ModelNoContextError
 from .ledger import Ledger
@@ -229,7 +229,7 @@ def run_model(self,
     @abstractmethod
     def run_model(self,
                   slug: str,
-                  input: Union[dict, DTOType] = EmptyInput(),
+                  input: Union[dict, DTOType],
                   return_type: Union[Type[dict], None] = None,
                   block_number: Union[int, None] = None,
                   version: Union[str, None] = None,
@@ -239,7 +239,7 @@ def run_model(self,
     @abstractmethod
     def run_model(self,  # type: ignore
                   slug,
-                  input=EmptyInput(),
+                  input,
                   return_type=None,
                   block_number=None,
                   version=None,

credmark/cmf/model/utils/historical_util.py

@@ -204,7 +204,7 @@ def parse_timerangestr(self, time_str: str):
         except Exception as _err:
             raise ModelRunError(
                 f"Invalid historical time string '{time_str}': "
-                f"unknown number format. {_err}")
+                f"unknown number format. {_err}") from None
         return (key, num)
 
     def range_timestamp(self, key: str, num: int):

credmark/cmf/types/address.py

@@ -14,7 +14,7 @@ def validate_address(addr: str) -> ChecksumAddress:
         eth_utils_validate_address(checksum_addr)
         return checksum_addr
     except Exception as e:
-        raise ModelTypeError(f'Address validation error: {str(e)}')
+        raise ModelTypeError(f'Address validation error: {str(e)}') from None
 
 
 evm_address_regex = re.compile(r'^0x[a-fA-F0-9]{40}$')
@@ -73,7 +73,7 @@ def __new__(cls, addr):
                 if len(addr) != 42:
                     addr = f'{int(addr, 16):#042x}'
             except Exception as e:
-                raise ModelTypeError(f'Address validation error: {str(e)}')
+                raise ModelTypeError(f'Address validation error: {str(e)}') from None
         elif isinstance(addr, bytes):
             addr = addr.hex()
             if len(addr) != 42:

credmark/cmf/types/adt.py

@@ -108,7 +108,7 @@ def to_dataframe(self):
         if len(self.records) == 0:
             return pd.DataFrame(columns=self.fields, data=[])
 
-        data_dict = [dict(zip(self.fields, r)) for r in self.records]
+        data_dict = [dict(zip(self.fields, r, strict=True)) for r in self.records]
         df_in = pd.DataFrame(data_dict)
         if len(self.fix_int_columns) > 0:
             for field in self.fix_int_columns:
@@ -117,8 +117,8 @@ def to_dataframe(self):
         return df_in
 
     @classmethod
-    def from_dataframe(cls, _df, fix_int_columns=[]):  # pylint:disable=dangerous-default-value
+    def from_dataframe(cls, _df, fix_int_columns=None):  # pylint:disable=dangerous-default-value
         return cls(records=list(_df.itertuples(index=False, name=None)),
                    fields=_df.columns.tolist(),
                    n_rows=_df.shape[0],
-                   fix_int_columns=fix_int_columns)
+                   fix_int_columns=[] if fix_int_columns is None else fix_int_columns)

credmark/cmf/types/network.py

@@ -63,9 +63,10 @@ class NetworkDict(DefaultDict[Network, T], Generic[T]):
     @staticmethod
     def _process_args(mapping=(), **kwargs):
         if hasattr(mapping, 'items'):
-            mapping = getattr(mapping, 'items')()
+            mapping = mapping.items()
+
         return ((Network.parse_network(k), v)
-                for k, v in chain(mapping, getattr(kwargs, 'items')()))
+                for k, v in chain(mapping, kwargs.items()))
 
     def __setitem__(self, k: NetworkKey, v: T):
         super().__setitem__(Network.parse_network(k), v)
@@ -92,6 +93,7 @@ def setdefault(self, k: NetworkKey, v: T):
         super().setdefault(Network.parse_network(k), v)
 
     def update(self, mapping: Iterable[tuple[NetworkKey, T]] = (), **kwargs):
+        print('update')
         super().update(self._process_args(mapping, **kwargs))
 
     @classmethod

credmark/dto/dto_schema.py

@@ -159,7 +159,7 @@ def dto_schema_viz(head_node,  # pylint: disable=too-many-arguments,too-many-loc
             return [{var_name: 'object'}]
 
     except Exception as err:
-        raise ValueError(f'Unknown schema node {var_name, node, err, tag}')
+        raise ValueError(f'Unknown schema node {var_name, node, err, tag}') from None
 
 
 def print_tree(tree, prefix, print_func):

credmark/dto/transform.py

@@ -75,4 +75,4 @@ def transform_data_for_dto(  # pylint: disable=too-many-return-statements
                 return dto_class(**data)
     except Exception as e:
         raise DataTransformError(
-            f'Error validating model {slug} {data_source}: {e}')
+            f'Error validating model {slug} {data_source}: {e}') from None

docs/ext/credmark_autosummary/__init__.py

@@ -311,7 +311,7 @@ def import_by_name(self, name: str, prefixes: List[str]) -> Tuple[str, Any, Any,
                     else:
                         errors = exc.exceptions + [exc2]
 
-                    raise ImportExceptionGroup(exc.args[0], errors)
+                    raise ImportExceptionGroup(exc.args[0], errors) from None
 
     def create_documenter(self, app: Sphinx, obj: Any,
                           parent: Any, full_name: str) -> "Documenter":
@@ -650,11 +650,15 @@ def get_import_prefixes_from_env(env: BuildEnvironment) -> List[Optional[str]]:
     return prefixes
 
 
-def import_by_name(name: str, prefixes: List[Optional[str]] = [None], grouped_exception: bool = False
+def import_by_name(name: str,
+                   prefixes: Optional[List[Optional[str]]] = None,
+                   grouped_exception: bool = False
                    ) -> Tuple[str, Any, Any, str]:
     """Import a Python object that has the given *name*, under one of the
     *prefixes*.  The first name that succeeds is used.
     """
+    prefixes = [None] if prefixes is None else prefixes
+
     tried = []
     errors: List[ImportExceptionGroup] = []
     for prefix in prefixes:
@@ -723,16 +727,18 @@ def _import_by_name(name: str, grouped_exception: bool = False) -> Tuple[Any, An
     except (ValueError, ImportError, AttributeError, KeyError) as exc:
         errors.append(exc)
         if grouped_exception:
-            raise ImportExceptionGroup('', errors)
+            raise ImportExceptionGroup('', errors) from None
         else:
             raise ImportError(*exc.args) from exc
 
 
-def import_ivar_by_name(name: str, prefixes: List[str] = [],
+def import_ivar_by_name(name: str, prefixes: Optional[List[str]] = None,
                         grouped_exception: bool = False) -> Tuple[str, Any, Any, str]:
     """Import an instance variable that has the given *name*, under one of the
     *prefixes*.  The first name that succeeds is used.
     """
+
+    prefixes = [] if prefixes is None else prefixes
     try:
         name, attr = name.rsplit(".", 1)
         real_name, obj, _parent, modname = import_by_name(name, prefixes, grouped_exception)

docs/ext/credmark_autosummary/generate.py

@@ -288,11 +288,13 @@ def get_all_members(obj: Any) -> Dict[str, Any]:
 
     def get_members(obj: Any,
                     types: Set[str],
-                    include_public: List[str] = [],
+                    include_public: Optional[List[str]] = None,
                     imported: bool = True) -> Tuple[List[str], List[str]]:
         items: List[str] = []
         public: List[str] = []
 
+        include_public = [] if include_public is None else include_public
+
         all_members = get_all_members(obj)
         for name, value in all_members.items():
             documenter = get_documenter(app, value, obj)

ruff.toml

@@ -1,3 +1,7 @@
 line-length = 120
+select = ["E", "F", "B",  "W", "I001"]
 ignore = ["E501"]
-select = ["F", "E", "W", "I001"]
+unfixable = ["B"]
+[per-file-ignores]
+"__init__.py" = ["E402"]
+"path/to/file.py" = ["E402"]

tests/token_transfer.md

@@ -0,0 +1,67 @@
+ERC20 transfer from logs
+
+```sql
+select
+count(distinct block_number, log_index, transaction_index, address, data, topics),
+max(block_number)
+from ethereum.core.logs
+where
+address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'
+and STARTSWITH(topics,'0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef,')
+-- and block_number <= 11055450
+limit 1;
+```
+
+
+```sql
+with double_entry_book as (
+    -- debits
+    select to_address as address, value as value, block_timestamp
+    from ethereum.traces
+    where to_address is not null
+    and status = 1
+    and (call_type not in ('delegatecall', 'callcode', 'staticcall') or call_type is null)
+    union all
+    -- credits
+    select from_address as address, -value as value, block_timestamp
+    from ethereum.traces
+    where from_address is not null
+    and status = 1
+    and (call_type not in ('delegatecall', 'callcode', 'staticcall') or call_type is null)
+    union all
+    -- transaction fees debits
+    select miner as address, sum(cast(gas as numeric) * cast(gas_price as numeric)) as value, block_timestamp
+    from ethereum.transactions as transactions
+    join ethereum.blocks as blocks on blocks.number = transactions.block_number
+    group by blocks.miner, block_timestamp
+    union all
+    -- transaction fees credits
+    select from_address as address, -(cast(gas as numeric) * cast(gas_price as numeric)) as value, block_timestamp
+    from ethereum.transactions
+),
+balances_by_address as (
+    select address, sum(value) as balance
+    from double_entry_book
+    group by address
+),
+balances_total AS (
+  SELECT SUM(value) AS total_balance
+  FROM double_entry_book
+),
+balances_by_address_top_million AS (
+  SELECT address, balance
+  FROM balances_by_address
+  ORDER BY balance DESC, address
+  LIMIT 1000000 OFFSET 0
+),
+cumm_balances_by_address_top_million AS (
+  SELECT address, balance, SUM(balance) OVER(ORDER BY balance DESC, address ASC) AS cumm_balance
+  FROM balances_by_address_top_million
+)
+SELECT address, balance, cumm_balance, (cumm_balance / total_balance) AS perc_total
+FROM cumm_balances_by_address_top_million
+INNER JOIN balances_total ON TRUE
+where address = lower('0x109B3C39d675A2FF16354E116d080B94d238a7c9')
+ORDER BY perc_total ASC
+;
+```