Config validators and logging

src/demo_quipuswap/handlers/on_fa12_divest_liquidity.py

@@ -1,4 +1,5 @@
 from decimal import Decimal
+
 import demo_quipuswap.models as models
 from demo_quipuswap.types.fa12_token.parameter.transfer import Transfer
 from demo_quipuswap.types.quipu_fa12.parameter.divest_liquidity import DivestLiquidity
@@ -22,4 +23,4 @@ async def on_fa12_divest_liquidity(
     transaction = next(op for op in ctx.operations if op.amount)
     position.tez_qty -= Decimal(transaction.amount) / (10 ** 6)  # type: ignore
     position.token_qty -= Decimal(transfer.parameter.value) / (10 ** decimals)
-    await position.save()
+    await position.save()

src/demo_quipuswap/handlers/on_fa12_invest_liquidity.py

@@ -1,4 +1,5 @@
 from decimal import Decimal
+
 import demo_quipuswap.models as models
 from demo_quipuswap.types.fa12_token.parameter.transfer import Transfer
 from demo_quipuswap.types.quipu_fa12.parameter.invest_liquidity import InvestLiquidity
@@ -21,4 +22,4 @@ async def on_fa12_invest_liquidity(
 
     position.tez_qty += Decimal(invest_liquidity.data.amount) / (10 ** 6)  # type: ignore
     position.token_qty += Decimal(transfer.parameter.value) / (10 ** decimals)
-    await position.save()
+    await position.save()

src/demo_quipuswap/handlers/on_fa20_divest_liquidity.py

@@ -1,4 +1,5 @@
 from decimal import Decimal
+
 import demo_quipuswap.models as models
 from demo_quipuswap.types.fa2_token.parameter.transfer import Transfer
 from demo_quipuswap.types.quipu_fa2.parameter.divest_liquidity import DivestLiquidity
@@ -22,4 +23,4 @@ async def on_fa20_divest_liquidity(
     transaction = next(op for op in ctx.operations if op.amount)
     position.tez_qty -= Decimal(transaction.amount) / (10 ** 6)  # type: ignore
     position.token_qty -= Decimal(transfer.parameter.__root__[0].txs[0].amount) / (10 ** decimals)
-    await position.save()
+    await position.save()

src/demo_quipuswap/handlers/on_fa20_invest_liquidity.py

@@ -1,4 +1,5 @@
 from decimal import Decimal
+
 import demo_quipuswap.models as models
 from demo_quipuswap.types.fa2_token.parameter.transfer import Transfer
 from demo_quipuswap.types.quipu_fa2.parameter.invest_liquidity import InvestLiquidity
@@ -21,4 +22,4 @@ async def on_fa20_invest_liquidity(
 
     position.tez_qty += Decimal(invest_liquidity.data.amount) / (10 ** 6)  # type: ignore
     position.token_qty += Decimal(transfer.parameter.__root__[0].txs[0].amount) / (10 ** decimals)
-    await position.save()
+    await position.save()

src/dipdup/config.py

@@ -9,19 +9,23 @@
 from os import environ as env
 from os.path import dirname
 from typing import Any, Callable, Dict, List, Optional, Type, Union
+from urllib.parse import urlparse
 
+from pydantic import validator
 from pydantic.dataclasses import dataclass
 from pydantic.json import pydantic_encoder
 from ruamel.yaml import YAML
 from tortoise import Tortoise
 from typing_extensions import Literal
 
+from dipdup.exceptions import ConfigurationError
 from dipdup.models import IndexType, State
 
 ROLLBACK_HANDLER = 'on_rollback'
 ENV_VARIABLE_REGEX = r'\${([\w]*):-(.*)}'
 
 sys.path.append(os.getcwd())
+_logger = logging.getLogger(__name__)
 
 
 def snake_to_camel(value: str) -> str:
@@ -92,6 +96,12 @@ def __hash__(self):
     def module_name(self) -> str:
         return self.typename if self.typename is not None else self.address
 
+    @validator('address')
+    def valid_address(cls, v):
+        if not v.startswith('KT1') or len(v) != 36:
+            raise ConfigurationError(f'`{v}` is not a valid contract address')
+        return v
+
 
 @dataclass
 class TzktDatasourceConfig:
@@ -107,6 +117,13 @@ class TzktDatasourceConfig:
     def __hash__(self):
         return hash(self.url)
 
+    @validator('url')
+    def valid_url(cls, v):
+        parsed_url = urlparse(v)
+        if not (parsed_url.scheme and parsed_url.netloc):
+            raise ConfigurationError(f'`{v}` is not a valid datasource URL')
+        return v
+
 
 @dataclass
 class OperationHandlerPatternConfig:
@@ -167,7 +184,7 @@ def __post_init_post_parse__(self):
     @property
     def callback_fn(self) -> Callable:
         if self._callback_fn is None:
-            raise Exception('Handler callable is not registered')
+            raise RuntimeError('Config is not initialized')
         return self._callback_fn
 
     @callback_fn.setter
@@ -208,7 +225,8 @@ def hash(self) -> str:
 
     @property
     def tzkt_config(self) -> TzktDatasourceConfig:
-        assert isinstance(self.datasource, TzktDatasourceConfig)
+        if not isinstance(self.datasource, TzktDatasourceConfig):
+            raise RuntimeError('Config is not initialized')
         return self.datasource
 
     @property
@@ -219,7 +237,7 @@ def contract_config(self) -> ContractConfig:
     @property
     def state(self):
         if not self._state:
-            raise Exception('Config is not initialized')
+            raise RuntimeError('Config is not initialized')
         return self._state
 
     @state.setter
@@ -229,7 +247,7 @@ def state(self, value: State):
     @property
     def rollback_fn(self) -> Callable:
         if not self._rollback_fn:
-            raise Exception('Config is not initialized')
+            raise RuntimeError('Config is not initialized')
         return self._rollback_fn
 
     @rollback_fn.setter
@@ -266,7 +284,8 @@ class BigmapdiffIndexConfig:
 
     @property
     def tzkt_config(self) -> TzktDatasourceConfig:
-        assert isinstance(self.datasource, TzktDatasourceConfig)
+        if not isinstance(self.datasource, TzktDatasourceConfig):
+            raise RuntimeError('Config is not initialized')
         return self.datasource
 
 
@@ -284,7 +303,8 @@ class BlockIndexConfig:
 
     @property
     def tzkt_config(self) -> TzktDatasourceConfig:
-        assert isinstance(self.datasource, TzktDatasourceConfig)
+        if not isinstance(self.datasource, TzktDatasourceConfig):
+            raise RuntimeError('Config is not initialized')
         return self.datasource
 
 
@@ -319,7 +339,7 @@ class DipDupConfig:
     database: Union[SqliteDatabaseConfig, DatabaseConfig] = SqliteDatabaseConfig(kind='sqlite')
 
     def __post_init_post_parse__(self):
-        self._logger = logging.getLogger(__name__)
+        _logger.info('Substituting index templates')
         for index_name, index_config in self.indexes.items():
             if isinstance(index_config, IndexTemplateConfig):
                 template = self.templates[index_config.template]
@@ -333,6 +353,7 @@ def __post_init_post_parse__(self):
 
         callback_patterns: Dict[str, List[List[OperationHandlerPatternConfig]]] = defaultdict(list)
 
+        _logger.info('Substituting contracts and datasources')
         for index_config in self.indexes.values():
             if isinstance(index_config, OperationIndexConfig):
                 if isinstance(index_config.datasource, str):
@@ -349,6 +370,7 @@ def __post_init_post_parse__(self):
             else:
                 raise NotImplementedError(f'Index kind `{index_config.kind}` is not supported')
 
+        _logger.info('Verifying callback uniqueness')
         for callback, patterns in callback_patterns.items():
             if len(patterns) > 1:
 
@@ -376,9 +398,11 @@ def load(
         current_workdir = os.path.join(os.getcwd())
         filename = os.path.join(current_workdir, filename)
 
+        _logger.info('Loading config from %s', filename)
         with open(filename) as file:
             raw_config = file.read()
 
+        _logger.info('Substituting environment variables')
         for match in re.finditer(ENV_VARIABLE_REGEX, raw_config):
             variable, default_value = match.group(1), match.group(2)
             value = env.get(variable)
@@ -390,13 +414,13 @@ def load(
         return config
 
     async def initialize(self) -> None:
-        self._logger.info('Setting up handlers and types for package `%s`', self.package)
+        _logger.info('Setting up handlers and types for package `%s`', self.package)
 
         rollback_fn = getattr(importlib.import_module(f'{self.package}.handlers.{ROLLBACK_HANDLER}'), ROLLBACK_HANDLER)
 
         for index_name, index_config in self.indexes.items():
             if isinstance(index_config, OperationIndexConfig):
-                self._logger.info('Getting state for index `%s`', index_name)
+                _logger.info('Getting state for index `%s`', index_name)
                 index_config.rollback_fn = rollback_fn
                 index_hash = index_config.hash()
                 state = await State.get_or_none(
@@ -412,20 +436,20 @@ async def initialize(self) -> None:
                     await state.save()
 
                 elif state.hash != index_hash:
-                    self._logger.warning('Config hash mismatch, reindexing')
+                    _logger.warning('Config hash mismatch, reindexing')
                     await Tortoise._drop_databases()
                     os.execl(sys.executable, sys.executable, *sys.argv)
 
                 index_config.state = state
 
                 for handler in index_config.handlers:
-                    self._logger.info('Registering handler callback `%s`', handler.callback)
+                    _logger.info('Registering handler callback `%s`', handler.callback)
                     handler_module = importlib.import_module(f'{self.package}.handlers.{handler.callback}')
                     callback_fn = getattr(handler_module, handler.callback)
                     handler.callback_fn = callback_fn
 
                     for pattern in handler.pattern:
-                        self._logger.info('Registering parameter type for entrypoint `%s`', pattern.entrypoint)
+                        _logger.info('Registering parameter type for entrypoint `%s`', pattern.entrypoint)
                         parameter_type_module = importlib.import_module(
                             f'{self.package}'
                             f'.types'
@@ -436,6 +460,7 @@ async def initialize(self) -> None:
                         parameter_type_cls = getattr(parameter_type_module, snake_to_camel(pattern.entrypoint))
                         pattern.parameter_type_cls = parameter_type_cls
 
+                        _logger.info('Registering storage type')
                         storage_type_module = importlib.import_module(
                             f'{self.package}' f'.types' f'.{pattern.contract_config.module_name}' f'.storage'
                         )

tests/test_dipdup/test_config.py

@@ -4,7 +4,8 @@
 
 from tortoise import Tortoise
 
-from dipdup.config import DipDupConfig
+from dipdup.config import ContractConfig, DipDupConfig, TzktDatasourceConfig
+from dipdup.exceptions import ConfigurationError
 
 
 class ConfigTest(IsolatedAsyncioTestCase):
@@ -33,3 +34,11 @@ async def test_load_initialize(self):
         )
         self.assertIsInstance(config.indexes['hen_mainnet'].handlers[0].callback_fn, Callable)
         self.assertIsInstance(config.indexes['hen_mainnet'].handlers[0].pattern[0].parameter_type_cls, Type)
+
+    async def test_validators(self):
+        with self.assertRaises(ConfigurationError):
+            ContractConfig(address='KT1lalala')
+        with self.assertRaises(ConfigurationError):
+            ContractConfig(address='lalalalalalalalalalalalalalalalalala')
+        with self.assertRaises(ConfigurationError):
+            TzktDatasourceConfig(kind='tzkt', url='not_an_url')

tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py

@@ -182,7 +182,7 @@ async def asyncSetUp(self):
         self.index_config = OperationIndexConfig(
             kind='operation',
             datasource='tzkt',
-            contract=ContractConfig(address='KT1lalala'),
+            contract=ContractConfig(address='KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9'),
             handlers=[
                 OperationHandlerConfig(
                     callback='',