diff --git a/poetry.lock b/poetry.lock index 499830a6c..de8fed0d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -49,6 +49,32 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "apscheduler" +version = "3.7.0" +description = "In-process task scheduler with Cron-like capabilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" + +[package.dependencies] +pytz = "*" +six = ">=1.4.0" +tzlocal = ">=2.0,<3.0" + +[package.extras] +asyncio = ["trollius"] +doc = ["sphinx", "sphinx-rtd-theme"] +gevent = ["gevent"] +mongodb = ["pymongo (>=3.0)"] +redis = ["redis (>=3.0)"] +rethinkdb = ["rethinkdb (>=2.4.0)"] +sqlalchemy = ["sqlalchemy (>=0.8)"] +testing = ["pytest (<6)", "pytest-cov", "pytest-tornado5", "mock", "pytest-asyncio (<0.6)", "pytest-asyncio"] +tornado = ["tornado (>=4.3)"] +twisted = ["twisted"] +zookeeper = ["kazoo"] + [[package]] name = "argcomplete" version = "1.12.3" @@ -214,7 +240,7 @@ wheel = ["wheel", "twine"] [[package]] name = "diff-cover" -version = "5.3.0" +version = "5.4.0" description = "Automatically find diff lines that need test coverage." category = "dev" optional = false @@ -711,7 +737,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] [[package]] name = "ruamel.yaml" -version = "0.17.9" +version = "0.17.10" description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" category = "main" optional = false @@ -726,7 +752,7 @@ jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] [[package]] name = "ruamel.yaml.clib" -version = "0.2.2" +version = "0.2.4" description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" category = "main" optional = false @@ -821,9 +847,20 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "tzlocal" +version = "2.1" +description = "tzinfo object for the local timezone" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pytz = "*" + [[package]] name = "urllib3" -version = "1.26.5" +version = "1.26.6" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false @@ -857,7 +894,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "a361b92403cc710754c02608ff9d7b494409cf67d8c17f2dae5a9251131f56b8" +content-hash = "ae34ad99d13490ed032c1edafde381362822a2a177c617ffb6bafed79574f04e" [metadata.files] aiohttp = [ @@ -911,6 +948,10 @@ appdirs = [ {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, ] +apscheduler = [ + {file = "APScheduler-3.7.0-py2.py3-none-any.whl", hash = "sha256:c06cc796d5bb9eb3c4f77727f6223476eb67749e7eea074d1587550702a7fbe3"}, + {file = "APScheduler-3.7.0.tar.gz", hash = "sha256:1cab7f2521e107d07127b042155b632b7a1cd5e02c34be5a28ff62f77c900c6a"}, +] argcomplete = [ {file = "argcomplete-1.12.3-py2.py3-none-any.whl", hash = "sha256:291f0beca7fd49ce285d2f10e4c1c77e9460cf823eef2de54df0c0fec88b0d81"}, {file = "argcomplete-1.12.3.tar.gz", hash = "sha256:2c7dbffd8c045ea534921e63b0be6fe65e88599990d8dc408ac8c542b72a5445"}, @@ -1026,8 +1067,8 @@ datamodel-code-generator = [ {file = "datamodel_code_generator-0.11.8-py3-none-any.whl", hash = "sha256:cd0a90bc81612e1b04b7b60e9321c73dc4e1122f05eaa657e645a03ed41be6c0"}, ] diff-cover = [ - {file = "diff_cover-5.3.0-py3-none-any.whl", hash = "sha256:f22db93b94a7ba956be6ee1c471763e6bcb6320f8e0293a401f037b7c778e86c"}, - {file = "diff_cover-5.3.0.tar.gz", hash = "sha256:82cc8c7a911ffe8e30af80bf1b15dd08180b84751179f5903516f522586cdfc8"}, + {file = "diff_cover-5.4.0-py3-none-any.whl", hash = "sha256:d4fbc19245c8648b01917905610ad174b7bbcced44b6c8f16f6a99c4ac126037"}, + {file = "diff_cover-5.4.0.tar.gz", hash = "sha256:e2243dfd071787f956f0713ac0565673a639ef1840116bb64d09c851924d00cb"}, ] dnspython = [ {file = "dnspython-2.1.0-py3-none-any.whl", hash = "sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216"}, @@ -1397,41 +1438,31 @@ requests = [ {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, ] "ruamel.yaml" = [ - {file = "ruamel.yaml-0.17.9-py3-none-any.whl", hash = "sha256:8873a6f5516e0d848c92418b0b006519c0566b6cd0dcee7deb9bf399e2bd204f"}, - {file = "ruamel.yaml-0.17.9.tar.gz", hash = "sha256:374373b4743aee9f6d9f40bea600fe020a7ac7ae36b838b4a6a93f72b584a14c"}, + {file = "ruamel.yaml-0.17.10-py3-none-any.whl", hash = "sha256:ffb9b703853e9e8b7861606dfdab1026cf02505bade0653d1880f4b2db47f815"}, + {file = "ruamel.yaml-0.17.10.tar.gz", hash = "sha256:106bc8d6dc6a0ff7c9196a47570432036f41d556b779c6b4e618085f57e39e67"}, ] "ruamel.yaml.clib" = [ - {file = "ruamel.yaml.clib-0.2.2-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:28116f204103cb3a108dfd37668f20abe6e3cafd0d3fd40dba126c732457b3cc"}, - {file = "ruamel.yaml.clib-0.2.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:daf21aa33ee9b351f66deed30a3d450ab55c14242cfdfcd377798e2c0d25c9f1"}, - {file = "ruamel.yaml.clib-0.2.2-cp27-cp27m-win32.whl", hash = "sha256:30dca9bbcbb1cc858717438218d11eafb78666759e5094dd767468c0d577a7e7"}, - {file = "ruamel.yaml.clib-0.2.2-cp27-cp27m-win_amd64.whl", hash = "sha256:f6061a31880c1ed6b6ce341215336e2f3d0c1deccd84957b6fa8ca474b41e89f"}, - {file = "ruamel.yaml.clib-0.2.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win32.whl", hash = "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f"}, - {file = "ruamel.yaml.clib-0.2.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win32.whl", hash = "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2"}, - {file = "ruamel.yaml.clib-0.2.2-cp36-cp36m-win_amd64.whl", hash = "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win32.whl", hash = "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6"}, - {file = "ruamel.yaml.clib-0.2.2-cp37-cp37m-win_amd64.whl", hash = "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win32.whl", hash = "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1"}, - {file = "ruamel.yaml.clib-0.2.2-cp38-cp38-win_amd64.whl", hash = "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win32.whl", hash = "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd"}, - {file = "ruamel.yaml.clib-0.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb"}, - {file = "ruamel.yaml.clib-0.2.2.tar.gz", hash = "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7"}, + {file = "ruamel.yaml.clib-0.2.4-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:329ac9064c1cfff9fc77fbecd90d07d698176fcd0720bfef9c2d27faa09dcc0e"}, + {file = "ruamel.yaml.clib-0.2.4-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:091a38f04f8a332ba7b3dba26197cd522bc29936943b3d1732ce3c463bb6b275"}, + {file = "ruamel.yaml.clib-0.2.4-cp35-cp35m-win32.whl", hash = "sha256:650cc8e65e2568fac84dc14970a09fe21b013a90621fff1626ea6d656cc03dc4"}, + {file = "ruamel.yaml.clib-0.2.4-cp35-cp35m-win_amd64.whl", hash = "sha256:729869106d5b7eb5e0260f7da4fcfef2cd9b324729fadc08edc27b1e86ad3013"}, + {file = "ruamel.yaml.clib-0.2.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:2ae2f58c18991c8565d41018177548a91c2f1511d8a185254632388f142fbae9"}, + {file = "ruamel.yaml.clib-0.2.4-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:c8a04c3f62a0b6a2696d003dd30e96e0b9d4a5ff450fe359c39a4a7466b9b935"}, + {file = "ruamel.yaml.clib-0.2.4-cp36-cp36m-win32.whl", hash = "sha256:fd400bd19ea3e86bad9fb5176ab7efb6efb5e440cc2fd435c86de021620d8fa7"}, + {file = "ruamel.yaml.clib-0.2.4-cp36-cp36m-win_amd64.whl", hash = "sha256:b1772bff158f785085ebc8e635a0b9450f0072413bc89d8fc7f0ee803d1ab7f8"}, + {file = "ruamel.yaml.clib-0.2.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3271fb4a379050735f90177d1e61b5cc9acb5130baf995f3c775fa2aa2b113fb"}, + {file = "ruamel.yaml.clib-0.2.4-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:aa157cee912030d8abfb97b278295abbb7923dedfd892f2e94c22adbf5730398"}, + {file = "ruamel.yaml.clib-0.2.4-cp37-cp37m-win32.whl", hash = "sha256:202e4751f038383241036e79640e7efd23d7272e3ce0cc8a11b9804ad604c5da"}, + {file = "ruamel.yaml.clib-0.2.4-cp37-cp37m-win_amd64.whl", hash = "sha256:3e506603394f5a678e9b924324bc1352c0493d7010ab4df687eb6d868631f9fb"}, + {file = "ruamel.yaml.clib-0.2.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9f95ae85986b53d6d0d253d570a9bb3a229e5319f1f76b2ba7809fa86cad890"}, + {file = "ruamel.yaml.clib-0.2.4-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2b9a62080d18c7fa17443e37f0d941d1be0a66ddcf5be5253f91cc59a15a9c1e"}, + {file = "ruamel.yaml.clib-0.2.4-cp38-cp38-win32.whl", hash = "sha256:769468005ce63bad78575b9d9f095f388ac1f45a331969e04135ac9626c3529d"}, + {file = "ruamel.yaml.clib-0.2.4-cp38-cp38-win_amd64.whl", hash = "sha256:83d72c5434151071cb67690be0034f9162ea282e58e47f9e8d23e8d14ca96584"}, + {file = "ruamel.yaml.clib-0.2.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:243941fe8f98053662f0394057b29d7146fe56e1b0011971302ea75e4b111529"}, + {file = "ruamel.yaml.clib-0.2.4-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:2d75c965c407fdef9d1b33cd39faf47aa106d3fa2cf83960ec9ed95c4c9a55bc"}, + {file = "ruamel.yaml.clib-0.2.4-cp39-cp39-win32.whl", hash = "sha256:f012b89c56f936e31f12a1484f08964c4681ae75488bc79c8909f37c517500f6"}, + {file = "ruamel.yaml.clib-0.2.4-cp39-cp39-win_amd64.whl", hash = "sha256:a6d8749819403338093c61ee897b97d0f4aa73297e97feb1705d143c002b5bed"}, + {file = "ruamel.yaml.clib-0.2.4.tar.gz", hash = "sha256:f997f13fd94e37e8b7d7dbe759088bb428adc6570da06b64a913d932d891ac8d"}, ] semver = [ {file = "semver-2.13.0-py2.py3-none-any.whl", hash = "sha256:ced8b23dceb22134307c1b8abfa523da14198793d9787ac838e70e29e77458d4"}, @@ -1490,9 +1521,13 @@ typing-extensions = [ {file = "typing_extensions-3.10.0.0-py3-none-any.whl", hash = "sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84"}, {file = "typing_extensions-3.10.0.0.tar.gz", hash = "sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342"}, ] +tzlocal = [ + {file = "tzlocal-2.1-py2.py3-none-any.whl", hash = "sha256:e2cb6c6b5b604af38597403e9852872d7f534962ae2954c7f35efcb1ccacf4a4"}, + {file = "tzlocal-2.1.tar.gz", hash = "sha256:643c97c5294aedc737780a49d9df30889321cbe1204eac2c2ec6134035a92e44"}, +] urllib3 = [ - {file = "urllib3-1.26.5-py2.py3-none-any.whl", hash = "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c"}, - {file = "urllib3-1.26.5.tar.gz", hash = "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"}, + {file = "urllib3-1.26.6-py2.py3-none-any.whl", hash = "sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4"}, + {file = "urllib3-1.26.6.tar.gz", hash = "sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"}, ] websockets = [ {file = "websockets-8.1-cp36-cp36m-macosx_10_6_intel.whl", hash = "sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c"}, diff --git a/pyproject.toml b/pyproject.toml index 8aaf53c1e..cfd886dbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ aiosignalrcore = "^0.9.2" fcache = "^0.4.7" click = "^8.0.1" pyee = "^8.1.0" +APScheduler = "^3.7.0" sentry-sdk = "^1.1.0" [tool.poetry.dev-dependencies] diff --git a/src/demo_hic_et_nunc/jobs/__init__.py b/src/demo_hic_et_nunc/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_quipuswap/jobs/__init__.py b/src/demo_quipuswap/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_registrydao/jobs/__init__.py b/src/demo_registrydao/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_tezos_domains/jobs/__init__.py b/src/demo_tezos_domains/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_tezos_domains_big_map/jobs/__init__.py b/src/demo_tezos_domains_big_map/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_tzbtc/jobs/__init__.py b/src/demo_tzbtc/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/demo_tzcolors/jobs/__init__.py b/src/demo_tzcolors/jobs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/dipdup/codegen.py b/src/dipdup/codegen.py index 0b83ceefe..c9a98006f 100644 --- a/src/dipdup/codegen.py +++ b/src/dipdup/codegen.py @@ -80,6 +80,13 @@ async def create_package(self) -> None: with open(join(handlers_path, '__init__.py'), 'w'): pass + self._logger.info('Creating `%s.jobs` package', self._config.package) + jobs_path = join(self._config.package_path, 'jobs') + with suppress(FileExistsError): + mkdir(jobs_path) + with open(join(jobs_path, '__init__.py'), 'w'): + pass + self._logger.info('Creating `%s/sql` directory', self._config.package) sql_path = join(self._config.package_path, 'sql') with suppress(FileExistsError): @@ -315,6 +322,23 @@ async def generate_user_handlers(self) -> None: else: raise NotImplementedError(f'Index kind `{index_config.kind}` is not supported') + async def generate_jobs(self) -> None: + if not self._config.jobs: + return + + jobs_path = join(self._config.package_path, 'jobs') + with open(join(dirname(__file__), 'templates', 'job.py.j2')) as file: + job_template = Template(file.read()) + + job_callbacks = set(job_config.callback for job_config in self._config.jobs.values()) + for job_callback in job_callbacks: + self._logger.info('Generating job `%s`', job_callback) + job_code = job_template.render(job=job_callback) + job_path = join(jobs_path, f'{job_callback}.py') + if not exists(job_path): + with open(job_path, 'w') as file: + file.write(job_code) + async def cleanup(self) -> None: """Remove fetched JSONSchemas""" self._logger.info('Cleaning up') diff --git a/src/dipdup/config.py b/src/dipdup/config.py index 201bfec15..499e0497e 100644 --- a/src/dipdup/config.py +++ b/src/dipdup/config.py @@ -426,6 +426,10 @@ class HandlerConfig: def __post_init_post_parse__(self): self._callback_fn = None + if self.callback in (ROLLBACK_HANDLER, CONFIGURE_HANDLER): + raise ConfigurationError(f'`{self.callback}` callback name is reserved') + if self.callback and self.callback != pascal_to_snake(self.callback): + raise ConfigurationError('`callback` field must conform to snake_case naming style') @property def callback_fn(self) -> Callable: @@ -610,6 +614,13 @@ def valid_url(cls, v): return v +@dataclass +class JobConfig(HandlerConfig): + crontab: str + args: Optional[Dict[str, Any]] = None + atomic: bool = False + + @dataclass class SentryConfig: dsn: str @@ -627,6 +638,7 @@ class DipDupConfig: :param templates: Mapping of template aliases and index templates :param database: Database config :param hasura: Hasura config + :param jobs: Mapping of job aliases and job configs :param sentry: Sentry integration config """ @@ -638,6 +650,7 @@ class DipDupConfig: templates: Optional[Dict[str, IndexConfigTemplateT]] = None database: Union[SqliteDatabaseConfig, PostgresDatabaseConfig] = SqliteDatabaseConfig(kind='sqlite') hasura: Optional[HasuraConfig] = None + jobs: Optional[Dict[str, JobConfig]] = None sentry: Optional[SentryConfig] = None def __post_init_post_parse__(self): @@ -833,6 +846,12 @@ def _initialize_handler_callback(self, handler_config: HandlerConfig) -> None: callback_fn = getattr(handler_module, handler_config.callback) handler_config.callback_fn = callback_fn + def _initialize_job_callback(self, job_config: JobConfig) -> None: + _logger.info('Registering job callback `%s`', job_config.callback) + job_module = importlib.import_module(f'{self.package}.jobs.{job_config.callback}') + callback_fn = getattr(job_module, job_config.callback) + job_config.callback_fn = callback_fn + def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None: if index_name in self._initialized: return @@ -890,12 +909,19 @@ def _initialize_index(self, index_name: str, index_config: IndexConfigT) -> None self._initialized.append(index_name) + def _initialize_jobs(self) -> None: + if not self.jobs: + return + for job_config in self.jobs.values(): + self._initialize_job_callback(job_config) + def initialize(self) -> None: _logger.info('Setting up handlers and types for package `%s`', self.package) self.pre_initialize() for index_name, index_config in self.indexes.items(): self._initialize_index(index_name, index_config) + self._initialize_jobs() @dataclass diff --git a/src/dipdup/dipdup.py b/src/dipdup/dipdup.py index 5087d76b3..3bac76721 100644 --- a/src/dipdup/dipdup.py +++ b/src/dipdup/dipdup.py @@ -1,10 +1,12 @@ import asyncio import hashlib import logging +from contextlib import suppress from os.path import join from posix import listdir from typing import Dict, List, cast +from apscheduler.schedulers import SchedulerNotRunningError # type: ignore from genericpath import exists from tortoise import Tortoise from tortoise.exceptions import OperationalError @@ -34,6 +36,7 @@ from dipdup.hasura import configure_hasura from dipdup.index import BigMapIndex, Index, OperationIndex from dipdup.models import BigMapData, IndexType, OperationData, State +from dipdup.scheduler import add_job, create_scheduler class IndexDispatcher: @@ -146,6 +149,7 @@ def __init__(self, config: DipDupConfig) -> None: datasources=self._datasources, ) self._index_dispatcher = IndexDispatcher(self._ctx) + self._scheduler = create_scheduler() async def init(self) -> None: """Create new or update existing dipdup project""" @@ -157,6 +161,7 @@ async def init(self) -> None: await codegen.generate_types() await codegen.generate_default_handlers() await codegen.generate_user_handlers() + await codegen.generate_jobs() await codegen.cleanup() for datasource in self._datasources.values(): @@ -180,6 +185,11 @@ async def run(self, reindex: bool, oneshot: bool) -> None: if self._config.hasura: worker_tasks.append(asyncio.create_task(configure_hasura(self._config))) + if self._config.jobs and not oneshot: + for job_name, job_config in self._config.jobs.items(): + add_job(self._ctx, self._scheduler, job_name, job_config) + self._scheduler.start() + worker_tasks.append(asyncio.create_task(self._index_dispatcher.run(oneshot))) try: @@ -189,6 +199,9 @@ async def run(self, reindex: bool, oneshot: bool) -> None: finally: self._logger.info('Closing datasource sessions') await asyncio.gather(*[d.close_session() for d in self._datasources.values()]) + # FIXME: AttributeError: 'NoneType' object has no attribute 'call_soon_threadsafe' + with suppress(AttributeError, SchedulerNotRunningError): + await self._scheduler.shutdown(wait=True) async def migrate(self) -> None: codegen = DipDupCodeGenerator(self._config, self._datasources_by_config) diff --git a/src/dipdup/scheduler.py b/src/dipdup/scheduler.py new file mode 100644 index 000000000..91ca4062c --- /dev/null +++ b/src/dipdup/scheduler.py @@ -0,0 +1,47 @@ +from apscheduler.executors.asyncio import AsyncIOExecutor # type: ignore +from apscheduler.jobstores.memory import MemoryJobStore # type: ignore +from apscheduler.schedulers.asyncio import AsyncIOScheduler # type: ignore +from apscheduler.triggers.cron import CronTrigger # type: ignore +from pytz import utc + +from dipdup.config import JobConfig +from dipdup.context import DipDupContext +from dipdup.utils import in_global_transaction + +jobstores = { + 'default': MemoryJobStore(), +} +executors = { + 'default': AsyncIOExecutor(), +} +job_defaults = { + 'coalesce': False, + 'max_instances': 3, +} + + +def create_scheduler() -> AsyncIOScheduler: + return AsyncIOScheduler( + jobstores=jobstores, + executors=executors, + job_defaults=job_defaults, + timezone=utc, + ) + + +def add_job(ctx: DipDupContext, scheduler: AsyncIOScheduler, job_name: str, job_config: JobConfig) -> None: + async def _atomic_wrapper(ctx, args): + async with in_global_transaction(): + await job_config.callback_fn(ctx, args) + + trigger = CronTrigger.from_crontab(job_config.crontab) + scheduler.add_job( + func=_atomic_wrapper if job_config.atomic else job_config.callback_fn, + id=job_name, + name=job_name, + trigger=trigger, + kwargs=dict( + ctx=ctx, + args=job_config.args, + ), + ) diff --git a/src/dipdup/templates/job.py.j2 b/src/dipdup/templates/job.py.j2 new file mode 100644 index 000000000..99d3a32ae --- /dev/null +++ b/src/dipdup/templates/job.py.j2 @@ -0,0 +1,8 @@ + +from typing import Any, Dict + +from dipdup.context import DipDupContext + + +async def {{job}}(ctx: DipDupContext, args: Dict[str, Any]) -> None: + ... diff --git a/tests/integration_tests/hic_et_nunc_job.yml b/tests/integration_tests/hic_et_nunc_job.yml new file mode 100644 index 000000000..aa9762a68 --- /dev/null +++ b/tests/integration_tests/hic_et_nunc_job.yml @@ -0,0 +1,59 @@ +spec_version: 1.0 +package: demo_hic_et_nunc + +database: + kind: sqlite + path: db.sqlite3 + +contracts: + HEN_objkts: + address: ${HEN_OBJKTS:-KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton} + typename: hen_objkts + HEN_minter: + address: ${HEN_MINTER:-KT1Hkg5qeNhfwpKW4fXvq7HGZB9z2EnmCCA9} + typename: hen_minter + +datasources: + tzkt_mainnet: + kind: tzkt + url: ${TZKT_URL:-https://api.tzkt.io} + +indexes: + hen_mainnet: + kind: operation + datasource: tzkt_mainnet + contracts: + - HEN_minter + handlers: + - callback: on_mint + pattern: + - type: transaction + destination: HEN_minter + entrypoint: mint_OBJKT + - type: transaction + destination: HEN_objkts + entrypoint: mint + - callback: on_swap + pattern: + - type: transaction + destination: HEN_minter + entrypoint: swap + - callback: on_cancel_swap + pattern: + - type: transaction + destination: HEN_minter + entrypoint: cancel_swap + - callback: on_collect + pattern: + - type: transaction + destination: HEN_minter + entrypoint: collect + first_block: 1365000 + last_block: 1366000 + +jobs: + test: + callback: test_job + crontab: "* * * * *" + args: + foo: bar \ No newline at end of file diff --git a/tests/integration_tests/test_codegen.py b/tests/integration_tests/test_codegen.py index 7e957084d..ec7bbeeec 100644 --- a/tests/integration_tests/test_codegen.py +++ b/tests/integration_tests/test_codegen.py @@ -32,7 +32,7 @@ def import_submodules(package, recursive=True): class CodegenTest(IsolatedAsyncioTestCase): async def test_codegen(self): for name in [ - 'hic_et_nunc.yml', + 'hic_et_nunc_job.yml', 'quipuswap.yml', 'tzcolors.yml', 'tezos_domains_big_map.yml', @@ -41,7 +41,7 @@ async def test_codegen(self): with self.subTest(name): config_path = join(dirname(__file__), name) config = DipDupConfig.load([config_path]) - config.initialize() + config.pre_initialize() config.package = 'tmp_test_dipdup' if config.package in sys.modules: