diff --git a/src/demo_hic_et_nunc/dipdup.yml b/src/demo_hic_et_nunc/dipdup.yml index ce5465554..ba2bb7ded 100644 --- a/src/demo_hic_et_nunc/dipdup.yml +++ b/src/demo_hic_et_nunc/dipdup.yml @@ -3,7 +3,7 @@ package: demo_hic_et_nunc database: kind: sqlite - path: db.sqlite3 + path: hic_et_nunc.sqlite3 contracts: HEN_objkts: diff --git a/src/demo_quipuswap/dipdup.yml b/src/demo_quipuswap/dipdup.yml index fb318ae7c..7a2d59048 100644 --- a/src/demo_quipuswap/dipdup.yml +++ b/src/demo_quipuswap/dipdup.yml @@ -3,7 +3,7 @@ package: demo_quipuswap database: kind: sqlite - path: db.sqlite3 + path: quipuswap.sqlite3 contracts: kusd_dex_mainnet: diff --git a/src/demo_quipuswap/hasura_metadata.json b/src/demo_quipuswap/hasura_metadata.json index be25b3cb0..f7fac3223 100644 --- a/src/demo_quipuswap/hasura_metadata.json +++ b/src/demo_quipuswap/hasura_metadata.json @@ -72,8 +72,8 @@ "role": "user", "permission": { "columns": [ - "tez_qty", "id", + "tez_qty", "token_qty" ], "filter": {}, @@ -107,13 +107,13 @@ "role": "user", "permission": { "columns": [ - "slippage", - "quantity", + "side", "level", + "slippage", "price", - "side", + "id", "timestamp", - "id" + "quantity" ], "filter": {}, "allow_aggregations": true diff --git a/src/dipdup/cli.py b/src/dipdup/cli.py index 46e0fd0b1..6fd13d288 100644 --- a/src/dipdup/cli.py +++ b/src/dipdup/cli.py @@ -20,6 +20,7 @@ from dipdup.config import DipDupConfig, IndexTemplateConfig, LoggingConfig, OperationIndexConfig, TzktDatasourceConfig from dipdup.datasources.tzkt.datasource import TzktDatasource from dipdup.models import IndexType, State +from dipdup.utils import tortoise_wrapper _logger = logging.getLogger(__name__) @@ -69,15 +70,10 @@ async def cli(ctx, config: str, logging_config: str): async def run(ctx) -> None: config: DipDupConfig = ctx.obj.config - try: + url = config.database.connection_string + models = f'{config.package}.models' + async with tortoise_wrapper(url, models): _logger.info('Initializing database') - await Tortoise.init( - db_url=config.database.connection_string, - modules={ - 'models': [f'{config.package}.models'], - 'int_models': ['dipdup.models'], - }, - ) connection_name, connection = next(iter(Tortoise._connections.items())) schema_sql = get_schema_sql(connection, False) @@ -118,9 +114,6 @@ async def run(ctx) -> None: datasource_run_tasks = [asyncio.create_task(d.start()) for d in datasources.values()] await asyncio.gather(*datasource_run_tasks) - finally: - await Tortoise.close_connections() - @cli.command(help='Initialize new dipdap') @click.pass_context diff --git a/src/dipdup/config.py b/src/dipdup/config.py index be6ce5984..527513a4e 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -432,6 +432,7 @@ async def initialize(self) -> None: index_name=index_name, index_type=IndexType.operation, hash=index_hash, + level=index_config.first_block - 1, ) await state.save() diff --git a/src/dipdup/configs/debug.yml b/src/dipdup/configs/debug.yml index 72e926e7c..7ae4cc608 100644 --- a/src/dipdup/configs/debug.yml +++ b/src/dipdup/configs/debug.yml @@ -13,13 +13,15 @@ SignalRCoreClient: formatter: brief dipdup.datasources.tzkt.datasource: - level: INFO + level: DEBUG dipdup.datasources.tzkt.cache: - level: INFO + level: DEBUG aiosqlite: - level: INFO + level: DEBUG db_client: - level: INFO + level: DEBUG + dipdup.models: + level: DEBUG root: level: DEBUG handlers: diff --git a/src/dipdup/configs/warning.yml b/src/dipdup/configs/warning.yml new file mode 100644 index 000000000..78d8d49bf --- /dev/null +++ b/src/dipdup/configs/warning.yml @@ -0,0 +1,26 @@ + version: 1 + disable_existing_loggers: false + formatters: + brief: + format: "%(levelname)-8s %(name)-35s %(message)s" + handlers: + console: + level: WARNING + formatter: brief + class: logging.StreamHandler + stream : ext://sys.stdout + loggers: + SignalRCoreClient: + formatter: brief + dipdup.datasources.tzkt.datasource: + level: INFO + dipdup.datasources.tzkt.cache: + level: INFO + aiosqlite: + level: INFO + db_client: + level: INFO + root: + level: WARNING + handlers: + - console \ No newline at end of file diff --git a/src/dipdup/datasources/tzkt/datasource.py b/src/dipdup/datasources/tzkt/datasource.py index e3ac61cbb..e9233685a 100644 --- a/src/dipdup/datasources/tzkt/datasource.py +++ b/src/dipdup/datasources/tzkt/datasource.py @@ -109,17 +109,24 @@ def _get_client(self) -> BaseHubConnection: async def start(self): self._logger.info('Starting datasource') + rest_only = False for operation_index_config in self._operation_index_configs.values(): - await self.add_subscription(operation_index_config.contract) + if operation_index_config.last_block: + await self.fetch_operations(operation_index_config.last_block, initial=True) + rest_only = True + continue + + await self.add_subscription(operation_index_config.contract) latest_block = await self.get_latest_block() current_level = latest_block['level'] state_level = operation_index_config.state.level if current_level != state_level: await self.fetch_operations(current_level, initial=True) - self._logger.info('Starting websocket client') - await self._get_client().start() + if not rest_only: + self._logger.info('Starting websocket client') + await self._get_client().start() async def stop(self): ... @@ -193,7 +200,7 @@ async def _process_operations(address, operations): for index_config in self._operation_index_configs.values(): sync_event = self._sync_events[index_config.state.index_name] - level = index_config.state.level or 0 + level = index_config.state.level operations = [] offset = 0 diff --git a/src/dipdup/models.py b/src/dipdup/models.py index fff3a07c9..a0d2bf7fb 100644 --- a/src/dipdup/models.py +++ b/src/dipdup/models.py @@ -81,23 +81,23 @@ def get_merged_storage(self, storage_type: Type[StorageType]) -> StorageType: if self.storage is None: raise Exception('`storage` field missing') - storage = self.storage - if self.bigmaps: - storage = deepcopy(self.storage) - _logger.debug('Merging storage') - _logger.debug('Before: %s', storage) - for key, field in storage_type.__fields__.items(): - # NOTE: TzKT could return bigmaps as object or as array of key-value objects. We need to guess this from storage. - # TODO: This code should be a part of datasource module. - if field.type_ not in (int, bool) and isinstance(storage[key], int): - if hasattr(field.type_, '__fields__') and 'key' in field.type_.__fields__ and 'value' in field.type_.__fields__: - storage[key] = [] + storage = deepcopy(self.storage) + _logger.debug('Merging storage') + _logger.debug('Before: %s', storage) + for key, field in storage_type.__fields__.items(): + # NOTE: TzKT could return bigmaps as object or as array of key-value objects. We need to guess this from storage. + # TODO: This code should be a part of datasource module. + if field.type_ not in (int, bool) and isinstance(storage[key], int): + if hasattr(field.type_, '__fields__') and 'key' in field.type_.__fields__ and 'value' in field.type_.__fields__: + storage[key] = [] + if self.bigmaps: self._merge_bigmapdiffs(storage, key, array=True) - else: - storage[key] = {} + else: + storage[key] = {} + if self.bigmaps: self._merge_bigmapdiffs(storage, key, array=False) - _logger.debug('After: %s', storage) + _logger.debug('After: %s', storage) return storage_type.parse_obj(storage) diff --git a/src/dipdup/utils.py b/src/dipdup/utils.py new file mode 100644 index 000000000..0f7d2d593 --- /dev/null +++ b/src/dipdup/utils.py @@ -0,0 +1,19 @@ +from contextlib import asynccontextmanager +from typing import Optional + +from tortoise import Tortoise + + +@asynccontextmanager +async def tortoise_wrapper(url: str, models: Optional[str] = None): + try: + modules = {'int_models': ['dipdup.models']} + if models: + modules['models'] = [models] + await Tortoise.init( + db_url=url, + modules=modules, # type: ignore + ) + yield + finally: + await Tortoise.close_connections() diff --git a/tests/integration_tests/hic_et_nunc.yml b/tests/integration_tests/hic_et_nunc.yml new file mode 100644 index 000000000..4f239786d --- /dev/null +++ b/tests/integration_tests/hic_et_nunc.yml @@ -0,0 +1,46 @@ +spec_version: 0.0.1 +package: demo_hic_et_nunc + +database: + kind: sqlite + path: db.sqlite3 + +contracts: + HEN_objkts: + address: ${HEN_OBJKTS:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton} + typename: hen_objkts + HEN_minter: + address: ${HEN_MINTER:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9} + typename: hen_minter + +datasources: + tzkt_mainnet: + kind: tzkt + url: ${TZKT_URL:-https://staging.api.tzkt.io} + +indexes: + hen_mainnet: + kind: operation + datasource: tzkt_mainnet + contract: HEN_minter + handlers: + - callback: on_mint + pattern: + - destination: HEN_minter + entrypoint: mint_OBJKT + - destination: HEN_objkts + entrypoint: mint + - callback: on_swap + pattern: + - destination: HEN_minter + entrypoint: swap + - callback: on_cancel_swap + pattern: + - destination: HEN_minter + entrypoint: cancel_swap + - callback: on_collect + pattern: + - destination: HEN_minter + entrypoint: collect + first_block: 1365000 + last_block: 1366000 \ No newline at end of file diff --git a/tests/integration_tests/quipuswap.yml b/tests/integration_tests/quipuswap.yml new file mode 100644 index 000000000..df6e55b5d --- /dev/null +++ b/tests/integration_tests/quipuswap.yml @@ -0,0 +1,121 @@ +spec_version: 0.0.1 +package: demo_quipuswap + +database: + kind: sqlite + path: db.sqlite3 + +contracts: + kusd_dex_mainnet: + address: KT1CiSKXR68qYSxnbzjwvfeMCRburaSDonT2 + typename: quipu_fa12 + tzbtc_dex_mainnet: + address: KT1N1wwNPqT5jGhM91GQ2ae5uY8UzFaXHMJS + typename: quipu_fa12 + kusd_token_mainnet: + address: KT1K9gCRgaLRFKTErYt1wVxA3Frb9FjasjTV + typename: fa12_token + tzbtc_token_mainnet: + address: KT1PWx2mnDueood7fEmfbBDKx1D9BAnnXitn + typename: fa12_token + hdao_dex_mainnet: + address: KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph + typename: quipu_fa2 + hdao_token_mainnet: + address: KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW + typename: fa2_token + +datasources: + tzkt_staging_mainnet: + kind: tzkt + url: https://staging.api.tzkt.io + +templates: + quipuswap_fa12: + kind: operation + datasource: tzkt_staging_mainnet + contract: + handlers: + - callback: on_fa12_token_to_tez + pattern: + - destination: + entrypoint: tokenToTezPayment + - destination: + entrypoint: transfer + - callback: on_fa12_tez_to_token + pattern: + - destination: + entrypoint: tezToTokenPayment + - destination: + entrypoint: transfer + - callback: on_fa12_invest_liquidity + pattern: + - destination: + entrypoint: investLiquidity + - destination: + entrypoint: transfer + - callback: on_fa12_divest_liquidity + pattern: + - destination: + entrypoint: divestLiquidity + - destination: + entrypoint: transfer + first_block: 1407528 + last_block: 1408934 + + quipuswap_fa2: + kind: operation + datasource: tzkt_staging_mainnet + contract: + handlers: + - callback: on_fa2_token_to_tez + pattern: + - destination: + entrypoint: tokenToTezPayment + - destination: + entrypoint: transfer + - callback: on_fa2_tez_to_token + pattern: + - destination: + entrypoint: tezToTokenPayment + - destination: + entrypoint: transfer + - callback: on_fa20_invest_liquidity + pattern: + - destination: + entrypoint: investLiquidity + - destination: + entrypoint: transfer + - callback: on_fa20_divest_liquidity + pattern: + - destination: + entrypoint: divestLiquidity + - destination: + entrypoint: transfer + first_block: 1407528 + last_block: 1407728 + +indexes: + kusd_mainnet: + template: quipuswap_fa12 + values: + dex_contract: kusd_dex_mainnet + token_contract: kusd_token_mainnet + symbol: kUSD + decimals: 18 + + # tzbtc_mainnet: + # template: quipuswap_fa12 + # values: + # dex_contract: tzbtc_dex_mainnet + # token_contract: tzbtc_token_mainnet + # symbol: tzBTC + # decimals: 8 + + hdao_mainnet: + template: quipuswap_fa2 + values: + dex_contract: hdao_dex_mainnet + token_contract: hdao_token_mainnet + symbol: hDAO + decimals: 6 diff --git a/tests/integration_tests/test_demos.py b/tests/integration_tests/test_demos.py new file mode 100644 index 000000000..35bb155d8 --- /dev/null +++ b/tests/integration_tests/test_demos.py @@ -0,0 +1,74 @@ +import subprocess +from os import mkdir +from os.path import dirname, join +from shutil import rmtree +from unittest import IsolatedAsyncioTestCase + +import demo_hic_et_nunc.models +import demo_quipuswap.models +import demo_tzcolors.models +from dipdup.utils import tortoise_wrapper + + +class DemosTest(IsolatedAsyncioTestCase): + def setUp(self): + mkdir('/tmp/dipdup') + + def tearDown(self): + rmtree('/tmp/dipdup') + + def run_dipdup(self, config: str): + subprocess.run( + [ + 'dipdup', + '-l', + 'warning.yml', + '-c', + join(dirname(__file__), config), + 'run', + ], + cwd='/tmp/dipdup', + check=True, + ) + + async def test_hic_et_nunc(self): + self.run_dipdup('hic_et_nunc.yml') + + async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_hic_et_nunc.models'): + holders = await demo_hic_et_nunc.models.Holder.filter().count() + tokens = await demo_hic_et_nunc.models.Token.filter().count() + swaps = await demo_hic_et_nunc.models.Swap.filter().count() + trades = await demo_hic_et_nunc.models.Trade.filter().count() + + self.assertEqual(22, holders) + self.assertEqual(29, tokens) + self.assertEqual(20, swaps) + self.assertEqual(24, trades) + + async def test_quipuswap(self): + self.run_dipdup('quipuswap.yml') + + async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_quipuswap.models'): + instruments = await demo_quipuswap.models.Instrument.filter().count() + traders = await demo_quipuswap.models.Trader.filter().count() + trades = await demo_quipuswap.models.Trade.filter().count() + positions = await demo_quipuswap.models.Position.filter().count() + + self.assertEqual(2, instruments) + self.assertEqual(73, traders) + self.assertEqual(94, trades) + self.assertEqual(56, positions) + + async def test_tzcolors(self): + self.run_dipdup('tzcolors.yml') + + async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_tzcolors.models'): + addresses = await demo_tzcolors.models.Address.filter().count() + tokens = await demo_tzcolors.models.Token.filter().count() + auctions = await demo_tzcolors.models.Auction.filter().count() + bids = await demo_tzcolors.models.Bid.filter().count() + + self.assertEqual(9, addresses) + self.assertEqual(14, tokens) + self.assertEqual(14, auctions) + self.assertEqual(44, bids) diff --git a/tests/integration_tests/tzcolors.yml b/tests/integration_tests/tzcolors.yml new file mode 100644 index 000000000..11de1c139 --- /dev/null +++ b/tests/integration_tests/tzcolors.yml @@ -0,0 +1,50 @@ +spec_version: 0.0.1 +package: demo_tzcolors + +database: + kind: sqlite + path: db.sqlite3 + +contracts: + tzcolors_minter: + address: KT1FyaDqiMQWg7Exo7VUiXAgZbd2kCzo3d4s + typename: tzcolors_minter + tzcolors_auction: + address: KT1CpeSQKdkhWi4pinYcseCFKmDhs5M74BkU + typename: tzcolors_auction + +datasources: + tzkt_staging: + kind: tzkt + url: ${TZKT_URL:-https://staging.api.tzkt.io} + +templates: + + tzcolors_auction: + kind: operation + datasource: + contract: + handlers: + - callback: on_create_auction + pattern: + - destination: + entrypoint: create_auction + - callback: on_bid + pattern: + - destination: + entrypoint: bid + - callback: on_withdraw + pattern: + - destination: + entrypoint: withdraw + first_block: 1335654 + last_block: 1340654 + +indexes: + + tzcolors_auction: + template: tzcolors_auction + values: + datasource: tzkt_staging + minter: tzcolors_minter + auction: tzcolors_auction \ No newline at end of file diff --git a/tests/test_dipdup/test_config.py b/tests/test_dipdup/test_config.py index 429b5befd..cd07ec69a 100644 --- a/tests/test_dipdup/test_config.py +++ b/tests/test_dipdup/test_config.py @@ -4,6 +4,8 @@ from tortoise import Tortoise +from dipdup.config import DipDupConfig +from dipdup.utils import tortoise_wrapper from dipdup.config import ContractConfig, DipDupConfig, TzktDatasourceConfig from dipdup.exceptions import ConfigurationError @@ -15,17 +17,9 @@ async def asyncSetUp(self): async def test_load_initialize(self): config = DipDupConfig.load(self.path) - try: - await Tortoise.init( - db_url='sqlite://:memory:', - modules={ - 'int_models': ['dipdup.models'], - }, - ) + async with tortoise_wrapper('sqlite://:memory:'): await Tortoise.generate_schemas() await config.initialize() - finally: - await Tortoise.close_connections() self.assertIsInstance(config, DipDupConfig) self.assertEqual( diff --git a/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py b/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py index bce436944..56b5f46dd 100644 --- a/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py +++ b/tests/test_dipdup/test_datasources/test_tzkt/test_datasource.py @@ -12,6 +12,7 @@ from dipdup.config import ContractConfig, OperationHandlerConfig, OperationHandlerPatternConfig, OperationIndexConfig from dipdup.datasources.tzkt.datasource import TzktDatasource from dipdup.models import HandlerContext, IndexType, OperationContext, OperationData, State +from dipdup.utils import tortoise_wrapper class Key(BaseModel): @@ -285,13 +286,7 @@ async def test_on_operation_message_data(self): on_operation_match_mock = AsyncMock() self.datasource.on_operation_match = on_operation_match_mock - try: - await Tortoise.init( - db_url='sqlite://:memory:', - modules={ - 'int_models': ['dipdup.models'], - }, - ) + async with tortoise_wrapper('sqlite://:memory:'): await Tortoise.generate_schemas() await self.datasource.on_operation_message([operations_message], self.index_config.contract.address, sync=True) @@ -303,22 +298,13 @@ async def test_on_operation_message_data(self): ANY, ) - finally: - await Tortoise.close_connections() - async def test_on_operation_match(self): with open(join(dirname(__file__), 'operations.json')) as file: operations_message = json.load(file) operations = [TzktDatasource.convert_operation(op) for op in operations_message['data']] matched_operation = operations[0] - try: - await Tortoise.init( - db_url='sqlite://:memory:', - modules={ - 'int_models': ['dipdup.models'], - }, - ) + async with tortoise_wrapper('sqlite://:memory:'): await Tortoise.generate_schemas() callback_mock = AsyncMock() @@ -336,9 +322,6 @@ async def test_on_operation_match(self): self.assertIsInstance(callback_mock.await_args[0][1].parameter, Collect) self.assertIsInstance(callback_mock.await_args[0][1].data, OperationData) - finally: - await Tortoise.close_connections() - async def test_on_operation_match_with_storage(self): with open(join(dirname(__file__), 'operations-storage.json')) as file: operations_message = json.load(file) @@ -347,13 +330,7 @@ async def test_on_operation_match_with_storage(self): operations = [TzktDatasource.convert_operation(op) for op in operations_message['data']] matched_operation = operations[0] - try: - await Tortoise.init( - db_url='sqlite://:memory:', - modules={ - 'int_models': ['dipdup.models'], - }, - ) + async with tortoise_wrapper('sqlite://:memory:'): await Tortoise.generate_schemas() callback_mock = AsyncMock() @@ -370,6 +347,3 @@ async def test_on_operation_match_with_storage(self): callback_mock.await_args[0][1].storage.proposals['e710c1a066bbbf73692168e783607996785260cec4d60930579827298493b8b9'], Proposals, ) - - finally: - await Tortoise.close_connections()