From efba0f35e71943b312db746b8f2c25df1e5a4b79 Mon Sep 17 00:00:00 2001 From: kdimentionaltree Date: Fri, 26 Aug 2022 12:09:37 +0200 Subject: [PATCH] Gigacommit with refactoring WIP Refactor of logic Refractoring of mytoninstaller.py Fixed inject_globals Removed print Fixed typos WIP WIP WIP WIP WIP2 WIP RC Fix Fixed scripts Crc16 -> fastcrc tonhttpapi installer Fixed merge WIP Split on three modules, added mypylib and mypyconsole submodules Added imports in mytoncore Updated mtc-jsonrpc installer update mtc-jsonrpc installer WIP Try fix merge conflicts Rebase. New commits apply 1. a61ab21 - add ubuntu 22.04 to the list of tested OS 2. a41ad04 - Update README.Ru.md 3. b12f234 - Add Traditional Chinese Support and Fix JSON Format Errors 4. d152a70 - Update README with tested operating systems and installation scripts 5. 329c6bd - Refactor wallet import documentation 6. 5d6afcd - feat: Add import wallet documentation 7. d00a94a - Refactor: Rename FAQ files to markdown format 8. e417285 - Add import-wallets.md file for importing wallets with private keys or mnemonic phrases. 9. 17b45b1 - Refactor directory usage and add error handling 10. 2458f8b - Add FAQ.md with instructions on how to install, remove and configure MyTonCtrl 11. 3748e9c - Refactor MyTonCtrl directory usage 12. dd5825d - Add instructions for using mytonctrl on Ubuntu 13. 559480d - Refactor installation instructions and add operability test 14. 4e41c94 - Refactor installation instructions for mytonctrl on Ubuntu 15. a9df737 - Refactor pool activation and deposit instructions 16. 4fb4a0f - Add Chinese translation for Nominator Pool documentation. 17. 47ad5cf - feat: Add instructions for running a validator node in nominator pool mode 18. c36fa7f - Refactor documentation for running a validator in nominator pool 19. b9c068f - Refactor Nominator Pool documentation for clarity 20. 95f6823 - Refactor README.md to include more detailed information 21. e52aa1c - Merge branch 'master' into update-documentation 22. f03095f - Add badges and fix formatting in README 22. cc711df - Refactor documentation structure and add MyTonCtrl installer mode 23. 4294fa8 - Refactor commit message: 24. 9d25122 - Update FAQ.md with improved formatting and content 25. 6f759c1 - Update Chinese translation of nominator-pool.md 26. 8c17c60 - Add stake param setting for nominator pool docs 27. 776cde0 - Fix tabs 28. 456b2dc - Merge remote-tracking branch 'sonofmom/feature/update_docs' into update-documentation 29. 9a993a7 - Add null stake option to usePool command 30. 2556501 - upgrade with ninja 31. ec3a8b9 - persist global.config.json before the upgrade 32. d51d20e - make persistence of global.config.json rights friendly 33. b3bc22e - clean up build directory (without removing it) prior to upgrade 34. 953e991 - avoid usage of /tmp dir on clean up 35. 8e14dec - Merge pull request #127 from Gusarich/master 36. e5978dd - Merge pull request #128 from awesome-doge/add-traditional-chinese-support 37. 4e97349 - Merge pull request #129 from awesome-doge/update-documentation 38. c1eee8d - Merge pull request #131 from neodiX42/upgrade-with-ninja 39. Applying commit efbf35c "update mypylib" Replay commit ef7d665 "bugfix" Replay commit 8733452 "upgrade bugfix" Replay commit 2a359a8 "update mypylib" Replay commits: 1. f1f5247 - add GetDomainFromAuction function 2. ac0d138 - Merge pull request #135 from ton-blockchain/master 3. 17fd02c - Merge pull request #136 from ton-blockchain/dev Replay commit dc56e83 "get_service_uptime bugfix on systems without systemd" Replay commits: 1. c3f3119 - mytoninstaller.py bugfix 2. 2c68e2f - bugfix if the files are in the wrong place Replay commit c789ebc "bugfix and moving to PEP8 stile" Replay commit 857b5a5 "bugfix" Replay commit 966f1a7 "bugfix" and update install.sh Refactor installation instructions for mytonctrl on Ubuntu The commit refactors the installation instructions for mytonctrl on Ubuntu. The changes include updating the language, adding numbered steps, and clarifying commands. Additionally, the commit adds a section to check wallet balances and activate wallets. Finally, it includes information about checking logs and using help commands. Refactor documentation for running a validator in nominator pool The commit message summarizes the changes made to the documentation for running a validator in a nominator pool. The changes include updating instructions on hardware requirements, recommended providers, and steps for creating and activating pools. Additionally, the commit includes instructions for switching from an individual validator to a nominator pool. Refactor Nominator Pool documentation for clarity This commit refactors the Nominator Pool documentation to improve its readability and clarity. Changes include rewording sentences, adding hyperlinks, and providing additional explanations for certain steps. The commit also includes a section on transitioning a regular validator to nominator pool mode. Refactor README.md to include more detailed information This commit refactors the README.md file to include a table of contents, installation instructions, and useful links. It also adds documentation in English, Russian, and Mandarin for frequently asked questions, importing wallets, manuals for Ubuntu, and nominator pools. Additionally, it includes information on telemetry and a web admin panel. Refactor documentation structure and add MyTonCtrl installer mode This commit refactors the documentation structure for MyTonCtrl, categorizing technical documents by language. It also adds a new feature, MyTonCtrl installer mode, which includes a web admin panel and local copy of toncenter. The web admin panel allows users to control the node/validator through their browser after installing an additional module. A local copy of toncenter can be set up on the user's server by enabling another module in the installer. add GetDomainFromAuction function Fix duplicated method GetDomainFromAuction --- .gitignore | 120 ++ .gitmodules | 4 +- README.Ru.md => README.RU.md | 0 README.md | 2 +- docs/en/FAQ.md | 2 +- docs/en/import-wallets.md | 5 +- docs/en/nominator-pool.md | 2 +- docs/ru/FAQ.md | 2 +- docs/ru/import-wallets.md | 4 +- docs/ru/nominator-pool.md | 2 +- docs/zh_TW/FAQ.md | 2 +- docs/zh_TW/import-wallets.md | 2 +- docs/zh_TW/manual-ubuntu.md | 2 +- docs/zh_TW/nominator-pool.md | 2 +- mytoncore/__init__.py | 4 + mytoncore/__main__.py | 5 + mytoncore/fift.py | 27 + mytoncore/functions.py | 585 +++++++ mytoncore/liteclient.py | 44 + mytoncore/models.py | 177 ++ mytoncore.py => mytoncore/mytoncore.py | 1418 +++++------------ mytoncore/tonblocksscanner.py | 208 +++ mytoncore/utils.py | 79 + mytoncore/validator_console.py | 27 + mytonctrl/__init__.py | 0 mytonctrl/__main__.py | 4 + mytonctrl.py => mytonctrl/mytonctrl.py | 576 +++++-- .../resources/translate.json | 0 {scripts => mytonctrl/scripts}/update.sh | 16 +- {scripts => mytonctrl/scripts}/upgrade.sh | 4 + {scripts => mytonctrl/scripts}/xrestart.py | 2 +- mytoninstaller/__init__.py | 0 mytoninstaller/__main__.py | 5 + mytoninstaller/config.py | 136 ++ mytoninstaller/mytoninstaller.py | 239 +++ mytoninstaller/scripts/__init__.py | 0 mytoninstaller/scripts/add2systemd.sh | 62 + .../scripts}/jsonrpcinstaller.sh | 16 +- .../scripts}/pytonv3installer.sh | 5 +- mytoninstaller/scripts/tonhttpapiinstaller.sh | 38 + .../settings.py | 610 ++----- mytoninstaller/utils.py | 33 + requirements.txt | 5 + scripts/install.sh | 98 +- scripts/{toninstaller.sh => ton_installer.sh} | 73 +- scripts/uninstall.sh | 33 +- scripts/upgrade.py | 31 - setup.py | 43 + tests/__init__.py | 0 tests/blocksScanner.py | 7 +- tests/bounce.py | 27 +- tests/mg.py | 4 +- tests/tpsLoad.py | 35 +- tests/tpsLoad2.py | 34 +- 54 files changed, 2975 insertions(+), 1886 deletions(-) rename README.Ru.md => README.RU.md (100%) create mode 100644 mytoncore/__init__.py create mode 100644 mytoncore/__main__.py create mode 100644 mytoncore/fift.py create mode 100755 mytoncore/functions.py create mode 100644 mytoncore/liteclient.py create mode 100644 mytoncore/models.py rename mytoncore.py => mytoncore/mytoncore.py (75%) mode change 100755 => 100644 create mode 100644 mytoncore/tonblocksscanner.py create mode 100644 mytoncore/utils.py create mode 100644 mytoncore/validator_console.py create mode 100644 mytonctrl/__init__.py create mode 100644 mytonctrl/__main__.py rename mytonctrl.py => mytonctrl/mytonctrl.py (69%) rename translate.json => mytonctrl/resources/translate.json (100%) rename {scripts => mytonctrl/scripts}/update.sh (73%) rename {scripts => mytonctrl/scripts}/upgrade.sh (92%) rename {scripts => mytonctrl/scripts}/xrestart.py (99%) create mode 100644 mytoninstaller/__init__.py create mode 100644 mytoninstaller/__main__.py create mode 100644 mytoninstaller/config.py create mode 100644 mytoninstaller/mytoninstaller.py create mode 100644 mytoninstaller/scripts/__init__.py create mode 100755 mytoninstaller/scripts/add2systemd.sh rename {scripts => mytoninstaller/scripts}/jsonrpcinstaller.sh (62%) mode change 100644 => 100755 rename {scripts => mytoninstaller/scripts}/pytonv3installer.sh (76%) mode change 100644 => 100755 create mode 100755 mytoninstaller/scripts/tonhttpapiinstaller.sh rename mytoninstaller.py => mytoninstaller/settings.py (61%) create mode 100644 mytoninstaller/utils.py create mode 100644 requirements.txt rename scripts/{toninstaller.sh => ton_installer.sh} (66%) mode change 100755 => 100644 delete mode 100644 scripts/upgrade.py create mode 100644 setup.py create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore index 7f752b37..4123b044 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,124 @@ +# Created by .ignore support plugin (hsz.mobi) +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +#PyCharm +.idea/ +.idea/$CACHE_FILE$ +.idea/.gitignore +.idea/encodings.xml +.idea/inspectionProfiles/ +.idea/misc.xml +.idea/modules.xml +.idea/ton_client.iml +.idea/vcs.xml + *.DS_Store .idea/ __pycache__/ test.py +.vscode/ +venv/ +venv38/ +sandbox/ diff --git a/.gitmodules b/.gitmodules index e6ac5652..fdb48b48 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,6 @@ [submodule "mypylib"] path = mypylib - url = https://github.com/igroman787/mypylib.git + url = https://github.com/igroman787/mypylib [submodule "mypyconsole"] path = mypyconsole - url = https://github.com/igroman787/mypyconsole.git + url = https://github.com/igroman787/mypyconsole diff --git a/README.Ru.md b/README.RU.md similarity index 100% rename from README.Ru.md rename to README.RU.md diff --git a/README.md b/README.md index a41b61d4..aaf1d762 100644 --- a/README.md +++ b/README.md @@ -155,4 +155,4 @@ Ready. A local copy of toncenter is available at `http://:800 git: https://github.com/igroman787/pytonv3 # Useful links -* https://docs.ton.org/ \ No newline at end of file +* https://docs.ton.org/ diff --git a/docs/en/FAQ.md b/docs/en/FAQ.md index d016e3a8..a0863b53 100644 --- a/docs/en/FAQ.md +++ b/docs/en/FAQ.md @@ -121,4 +121,4 @@ If you need to restart your validator, you can do so by running the following co systemctl restart validator ``` -Ensure you have sufficient permissions to execute these commands and make necessary adjustments. Always remember to back up important data before performing operations that could potentially affect your validator. \ No newline at end of file +Ensure you have sufficient permissions to execute these commands and make necessary adjustments. Always remember to back up important data before performing operations that could potentially affect your validator. diff --git a/docs/en/import-wallets.md b/docs/en/import-wallets.md index 508f2655..0fcac1c9 100644 --- a/docs/en/import-wallets.md +++ b/docs/en/import-wallets.md @@ -5,11 +5,9 @@ MyTonCtrl supports various types of wallet-like contracts, including wallet-v1, ## Importing Using a Private Key If you have access to a private key, you can easily import a wallet. Enter the following command into the console: - ``` iw ``` - Here, `` is your private key in base64 format. ## Importing Using a Mnemonic Phrase @@ -32,5 +30,4 @@ If you have a mnemonic phrase (a sequence of 24 words like `tattoo during ...`), 6. Open the mytonctrl console and list the wallets using the `wl` command. 7. Verify that the wallet has been imported and displays the correct balance. 8. You can now send funds using the `mg` command. Enter `mg` to view the help documentation. - -Remember to replace placeholders (words inside `< >`) with your actual values when running commands. \ No newline at end of file +Remember to replace placeholders (words inside `< >`) with your actual values when running commands. diff --git a/docs/en/nominator-pool.md b/docs/en/nominator-pool.md index fd80ee6d..4539f2bd 100644 --- a/docs/en/nominator-pool.md +++ b/docs/en/nominator-pool.md @@ -115,4 +115,4 @@ If you're creating a pool for numerous nominators, you might use something like 2. Await the return of both your stakes from the elector. -3. Proceed with the steps under "Running the Validator in Nominator Pool Mode" from the **4th step** onwards. \ No newline at end of file +3. Proceed with the steps under "Running the Validator in Nominator Pool Mode" from the **4th step** onwards. diff --git a/docs/ru/FAQ.md b/docs/ru/FAQ.md index e1436e1d..ddf57bec 100644 --- a/docs/ru/FAQ.md +++ b/docs/ru/FAQ.md @@ -119,4 +119,4 @@ Error: expected str, bytes or os.PathLike object, not NoneType systemctl restart validator ``` -Убедитесь, что у вас есть достаточные права для выполнения этих команд и сделайте необходимые корректировки. Всегда помните о резервном копировании важных данных перед выполнением операций, которые могут потенциально повлиять на ваш валидатор. \ No newline at end of file +Убедитесь, что у вас есть достаточные права для выполнения этих команд и сделайте необходимые корректировки. Всегда помните о резервном копировании важных данных перед выполнением операций, которые могут потенциально повлиять на ваш валидатор. diff --git a/docs/ru/import-wallets.md b/docs/ru/import-wallets.md index 700b7ec9..f61feab9 100644 --- a/docs/ru/import-wallets.md +++ b/docs/ru/import-wallets.md @@ -27,10 +27,10 @@ iw <адрес-кошелька> <секретный-ключ-кошелька> ``` node index.js word1 word2 ... word24 [address] ``` -4. Скрипт сгенерирует `wallet.pk` и `wallet.addr`. Переименуйте их в `imported_wallet.pk` и `imported_wallet.addr`. +4. Скрипт сгенерирует `wallet.pk` и `wallet.addr`. Переименуйте их в `imported_wallet.pk` и `imported_wallet.addr`. 5. Скопируйте оба файла в каталог `~/.local/share/mytoncore/wallets/`. 6. Откройте консоль mytonctrl и перечислите кошельки с помощью команды `wl`. 7. Убедитесь, что кошелек был импортирован и отображает правильный баланс. 8. Теперь вы можете отправить средства с помощью команды `mg`. Введите `mg`, чтобы просмотреть справочную документацию. -Помните, что при выполнении команд следует заменить заполнители (слова внутри `< >`) на ваши фактические значения. \ No newline at end of file +Помните, что при выполнении команд следует заменить заполнители (слова внутри `< >`) на ваши фактические значения. diff --git a/docs/ru/nominator-pool.md b/docs/ru/nominator-pool.md index a735f1b8..7d87b2b1 100644 --- a/docs/ru/nominator-pool.md +++ b/docs/ru/nominator-pool.md @@ -115,4 +115,4 @@ 2. Дождитесь, когда оба ваших депозита вернутся от электора. -3. Следуйте инструкциям "Запуск валидатора в режиме номинантского пула", начиная с **4-го шага**. \ No newline at end of file +3. Следуйте инструкциям "Запуск валидатора в режиме номинантского пула", начиная с **4-го шага**. diff --git a/docs/zh_TW/FAQ.md b/docs/zh_TW/FAQ.md index 54bfce6e..067902ed 100644 --- a/docs/zh_TW/FAQ.md +++ b/docs/zh_TW/FAQ.md @@ -118,4 +118,4 @@ Error: expected str, bytes or os.PathLike object, not NoneType systemctl restart validator ``` -請確保你具有執行這些命令的適當權限，並進行必要的調整。在執行可能影響你的驗證者的操作之前，請始終記得備份重要數據。 \ No newline at end of file +請確保你具有執行這些命令的適當權限，並進行必要的調整。在執行可能影響你的驗證者的操作之前，請始終記得備份重要數據。 diff --git a/docs/zh_TW/import-wallets.md b/docs/zh_TW/import-wallets.md index 8564ed68..1e199e1c 100644 --- a/docs/zh_TW/import-wallets.md +++ b/docs/zh_TW/import-wallets.md @@ -33,4 +33,4 @@ iw 7. 確認錢包已經匯入並且餘額正確。 8. 現在你可以使用 `mg` 命令發送金錢（輸入 `mg` 可查看使用說明）。 -在執行命令時，記得將尖括號內的佔位符（例如 ``、``）替換為實際的值。 \ No newline at end of file +在執行命令時，記得將尖括號內的佔位符（例如 ``、``）替換為實際的值。 diff --git a/docs/zh_TW/manual-ubuntu.md b/docs/zh_TW/manual-ubuntu.md index 4ab6da0a..aa6a98b7 100644 --- a/docs/zh_TW/manual-ubuntu.md +++ b/docs/zh_TW/manual-ubuntu.md @@ -64,4 +64,4 @@ 要檢查 **mytoncrl** 日誌，對於本地用戶，打開 `~/.local/share/mytoncore/mytoncore.log`，對於 Root，打開 `/usr/local/bin/mytoncore/mytoncore.log`。 -![logs](https://raw.githubusercontent.com/ton-blockchain/mytonctrl/master/screens/manual-ubuntu_mytoncore-log.png) \ No newline at end of file +![logs](https://raw.githubusercontent.com/ton-blockchain/mytonctrl/master/screens/manual-ubuntu_mytoncore-log.png) diff --git a/docs/zh_TW/nominator-pool.md b/docs/zh_TW/nominator-pool.md index e3fac6fe..fd107ca6 100644 --- a/docs/zh_TW/nominator-pool.md +++ b/docs/zh_TW/nominator-pool.md @@ -114,4 +114,4 @@ 2. 等待你的兩個賭注從選民那裡返回。 -3. 從**第四步**開始，按照"在提名人池模式下運行驗證者"的步驟進行操作。 \ No newline at end of file +3. 從**第四步**開始，按照"在提名人池模式下運行驗證者"的步驟進行操作。 diff --git a/mytoncore/__init__.py b/mytoncore/__init__.py new file mode 100644 index 00000000..a39eb41f --- /dev/null +++ b/mytoncore/__init__.py @@ -0,0 +1,4 @@ +from .utils import * +from .mytoncore import * +from mypylib.mypylib import MyPyClass +from mypyconsole.mypyconsole import MyPyConsole diff --git a/mytoncore/__main__.py b/mytoncore/__main__.py new file mode 100644 index 00000000..d905da8f --- /dev/null +++ b/mytoncore/__main__.py @@ -0,0 +1,5 @@ +from mytoncore.functions import mytoncore + + +if __name__ == '__main__': + mytoncore() diff --git a/mytoncore/fift.py b/mytoncore/fift.py new file mode 100644 index 00000000..291b3707 --- /dev/null +++ b/mytoncore/fift.py @@ -0,0 +1,27 @@ +import subprocess + + +class Fift: + def __init__(self, local): + self.local = local + self.appPath = None + self.libsPath = None + self.smartcontsPath = None + #end define + + def Run(self, args, **kwargs): + fift_timeout = self.local.db.fift_timeout if self.local.db.fift_timeout else 3 + timeout = kwargs.get("timeout", fift_timeout) + for i in range(len(args)): + args[i] = str(args[i]) + includePath = self.libsPath + ':' + self.smartcontsPath + args = [self.appPath, "-I", includePath, "-s"] + args + process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + output = process.stdout.decode("utf-8") + err = process.stderr.decode("utf-8") + if len(err) > 0: + self.local.add_log("args: {args}".format(args=args), "error") + raise Exception("Fift error: {err}".format(err=err)) + return output + #end define +#end class diff --git a/mytoncore/functions.py b/mytoncore/functions.py new file mode 100755 index 00000000..ac40b572 --- /dev/null +++ b/mytoncore/functions.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# -*- coding: utf_8 -*-l +import os +import sys +import psutil +import time +import json +import requests +import subprocess + +from mytoncore.mytoncore import MyTonCore, Dec2HexAddr +from mytoncore.tonblocksscanner import TonBlocksScanner +from mypylib.mypylib import ( + b2mb, + get_timestamp, + get_internet_interface_name, + get_service_pid, + get_load_avg, + thr_sleep, + Dict +) + + +def Init(local): + # Event reaction + if ("-e" in sys.argv): + x = sys.argv.index("-e") + eventName = sys.argv[x+1] + Event(local, eventName) + # end if + + local.run() + + # statistics + local.buffer.blocksData = dict() + local.buffer.transData = dict() + local.buffer.network = [None]*15*6 + local.buffer.diskio = [None]*15*6 + + # scan blocks + local.buffer.masterBlocksList = list() + local.buffer.prevShardsBlock = dict() + local.buffer.blocksNum = 0 + local.buffer.transNum = 0 +# end define + + +def Event(local, eventName): + if eventName == "enableVC": + EnableVcEvent(local) + elif eventName == "validator down": + ValidatorDownEvent(local) + local.exit() +# end define + + +def EnableVcEvent(local): + local.add_log("start EnableVcEvent function", "debug") + # Создать новый кошелек для валидатора + ton = MyTonCore(local) + wallet = ton.CreateWallet("validator_wallet_001", -1) + local.db["validatorWalletName"] = wallet.name + + # Создать новый ADNL адрес для валидатора + adnlAddr = ton.CreateNewKey() + ton.AddAdnlAddrToValidator(adnlAddr) + local.db["adnlAddr"] = adnlAddr + + # Сохранить + local.save() +# end define + + +def ValidatorDownEvent(local): + local.add_log("start ValidatorDownEvent function", "debug") + local.add_log("Validator is down", "error") +# end define + + +def Elections(local, ton): + usePool = local.db.get("usePool") + if usePool == True: + ton.PoolsUpdateValidatorSet() + ton.RecoverStake() + ton.ElectionEntry() + else: + ton.RecoverStake() + ton.ElectionEntry() +# end define + + +def Statistics(local): + ReadNetworkData(local) + SaveNetworkStatistics(local) + # ReadTransData(local, scanner) + SaveTransStatistics(local) + ReadDiskData(local) + SaveDiskStatistics(local) +# end define + + +def ReadDiskData(local): + timestamp = get_timestamp() + disks = GetDisksList() + buff = psutil.disk_io_counters(perdisk=True) + data = dict() + for name in disks: + data[name] = dict() + data[name]["timestamp"] = timestamp + data[name]["busyTime"] = buff[name].busy_time + data[name]["readBytes"] = buff[name].read_bytes + data[name]["writeBytes"] = buff[name].write_bytes + data[name]["readCount"] = buff[name].read_count + data[name]["writeCount"] = buff[name].write_count + # end for + + local.buffer.diskio.pop(0) + local.buffer.diskio.append(data) +# end define + + +def SaveDiskStatistics(local): + data = local.buffer.diskio + data = data[::-1] + zerodata = data[0] + buff1 = data[1*6-1] + buff5 = data[5*6-1] + buff15 = data[15*6-1] + if buff5 is None: + buff5 = buff1 + if buff15 is None: + buff15 = buff5 + # end if + + disksLoadAvg = dict() + disksLoadPercentAvg = dict() + iopsAvg = dict() + disks = GetDisksList() + for name in disks: + if zerodata[name]["busyTime"] == 0: + continue + diskLoad1, diskLoadPercent1, iops1 = CalculateDiskStatistics( + zerodata, buff1, name) + diskLoad5, diskLoadPercent5, iops5 = CalculateDiskStatistics( + zerodata, buff5, name) + diskLoad15, diskLoadPercent15, iops15 = CalculateDiskStatistics( + zerodata, buff15, name) + disksLoadAvg[name] = [diskLoad1, diskLoad5, diskLoad15] + disksLoadPercentAvg[name] = [diskLoadPercent1, + diskLoadPercent5, diskLoadPercent15] + iopsAvg[name] = [iops1, iops5, iops15] + # end fore + + # save statistics + statistics = local.db.get("statistics", dict()) + statistics["disksLoadAvg"] = disksLoadAvg + statistics["disksLoadPercentAvg"] = disksLoadPercentAvg + statistics["iopsAvg"] = iopsAvg + local.db["statistics"] = statistics +# end define + + +def CalculateDiskStatistics(zerodata, data, name): + if data is None: + return None, None, None + data = data[name] + zerodata = zerodata[name] + timeDiff = zerodata["timestamp"] - data["timestamp"] + busyTimeDiff = zerodata["busyTime"] - data["busyTime"] + diskReadDiff = zerodata["readBytes"] - data["readBytes"] + diskWriteDiff = zerodata["writeBytes"] - data["writeBytes"] + diskReadCountDiff = zerodata["readCount"] - data["readCount"] + diskWriteCountDiff = zerodata["writeCount"] - data["writeCount"] + diskLoadPercent = busyTimeDiff / 1000 / timeDiff * \ + 100 # /1000 - to second, *100 - to percent + diskLoadPercent = round(diskLoadPercent, 2) + diskRead = diskReadDiff / timeDiff + diskWrite = diskWriteDiff / timeDiff + diskReadCount = diskReadCountDiff / timeDiff + diskWriteCount = diskWriteCountDiff / timeDiff + diskLoad = b2mb(diskRead + diskWrite) + iops = round(diskReadCount + diskWriteCount, 2) + return diskLoad, diskLoadPercent, iops +# end define + + +def GetDisksList(): + data = list() + buff = os.listdir("/sys/block/") + for item in buff: + if "loop" in item: + continue + data.append(item) + # end for + data.sort() + return data +# end define + + +def ReadNetworkData(local): + timestamp = get_timestamp() + interfaceName = get_internet_interface_name() + buff = psutil.net_io_counters(pernic=True) + buff = buff[interfaceName] + data = dict() + data["timestamp"] = timestamp + data["bytesRecv"] = buff.bytes_recv + data["bytesSent"] = buff.bytes_sent + data["packetsSent"] = buff.packets_sent + data["packetsRecv"] = buff.packets_recv + + local.buffer.network.pop(0) + local.buffer.network.append(data) +# end define + + +def SaveNetworkStatistics(local): + data = local.buffer.network + data = data[::-1] + zerodata = data[0] + buff1 = data[1*6-1] + buff5 = data[5*6-1] + buff15 = data[15*6-1] + if buff5 is None: + buff5 = buff1 + if buff15 is None: + buff15 = buff5 + # end if + + netLoadAvg = dict() + ppsAvg = dict() + networkLoadAvg1, ppsAvg1 = CalculateNetworkStatistics(zerodata, buff1) + networkLoadAvg5, ppsAvg5 = CalculateNetworkStatistics(zerodata, buff5) + networkLoadAvg15, ppsAvg15 = CalculateNetworkStatistics(zerodata, buff15) + netLoadAvg = [networkLoadAvg1, networkLoadAvg5, networkLoadAvg15] + ppsAvg = [ppsAvg1, ppsAvg5, ppsAvg15] + + # save statistics + statistics = local.db.get("statistics", dict()) + statistics["netLoadAvg"] = netLoadAvg + statistics["ppsAvg"] = ppsAvg + local.db["statistics"] = statistics +# end define + + +def CalculateNetworkStatistics(zerodata, data): + if data is None: + return None, None + timeDiff = zerodata["timestamp"] - data["timestamp"] + bytesRecvDiff = zerodata["bytesRecv"] - data["bytesRecv"] + bytesSentDiff = zerodata["bytesSent"] - data["bytesSent"] + packetsRecvDiff = zerodata["packetsRecv"] - data["packetsRecv"] + packetsSentDiff = zerodata["packetsSent"] - data["packetsSent"] + bitesRecvAvg = bytesRecvDiff / timeDiff * 8 + bitesSentAvg = bytesSentDiff / timeDiff * 8 + packetsRecvAvg = packetsRecvDiff / timeDiff + packetsSentAvg = packetsSentDiff / timeDiff + netLoadAvg = b2mb(bitesRecvAvg + bitesSentAvg) + ppsAvg = round(packetsRecvAvg + packetsSentAvg, 2) + return netLoadAvg, ppsAvg +# end define + + +def ReadTransData(local, scanner): + transData = local.buffer.transData + SetToTimeData(transData, scanner.transNum) + ShortTimeData(transData) +# end define + + +def SetToTimeData(timeDataList, data): + timenow = int(time.time()) + timeDataList[timenow] = data +# end define + + +def ShortTimeData(data, max=120, diff=20): + if len(data) < max: + return + buff = data.copy() + data.clear() + keys = sorted(buff.keys(), reverse=True) + for item in keys[:max-diff]: + data[item] = buff[item] +# end define + + +def SaveTransStatistics(local): + tps1 = GetTps(local, 60) + tps5 = GetTps(local, 60*5) + tps15 = GetTps(local, 60*15) + + # save statistics + statistics = local.db.get("statistics", dict()) + statistics["tpsAvg"] = [tps1, tps5, tps15] + local.db["statistics"] = statistics +# end define + + +def GetDataPerSecond(data, timediff): + if len(data) == 0: + return + timenow = sorted(data.keys())[-1] + now = data.get(timenow) + prev = GetItemFromTimeData(data, timenow-timediff) + if prev is None: + return + diff = now - prev + result = diff / timediff + result = round(result, 2) + return result +# end define + + +def GetItemFromTimeData(data, timeneed): + if timeneed in data: + result = data.get(timeneed) + else: + result = data[min(data.keys(), key=lambda k: abs(k-timeneed))] + return result +# end define + + +def GetTps(local, timediff): + data = local.buffer.transData + tps = GetDataPerSecond(data, timediff) + return tps +# end define + + +def GetBps(local, timediff): + data = local.buffer.blocksData + bps = GetDataPerSecond(data, timediff) + return bps +# end define + + +def GetBlockTimeAvg(local, timediff): + bps = GetBps(local, timediff) + if bps is None or bps == 0: + return + result = 1/bps + result = round(result, 2) + return result +# end define + + +def Offers(local, ton): + saveOffers = ton.GetSaveOffers() + offers = ton.GetOffers() + for offer in offers: + offerHash = offer.get("hash") + if offerHash in saveOffers: + ton.VoteOffer(offerHash) +# end define + + +def Domains(local, ton): + pass +# end define + + +def GetUname(): + data = os.uname() + result = dict( + zip('sysname nodename release version machine'.split(), data)) + result.pop("nodename") + return result +# end define + + +def GetMemoryInfo(): + result = dict() + data = psutil.virtual_memory() + result["total"] = round(data.total / 10**9, 2) + result["usage"] = round(data.used / 10**9, 2) + result["usagePercent"] = data.percent + return result +# end define + + +def GetSwapInfo(): + result = dict() + data = psutil.swap_memory() + result["total"] = round(data.total / 10**9, 2) + result["usage"] = round(data.used / 10**9, 2) + result["usagePercent"] = data.percent + return result +# end define + + +def GetValidatorProcessInfo(): + pid = get_service_pid("validator") + if pid == None or pid == 0: + return + p = psutil.Process(pid) + mem = p.memory_info() + result = dict() + result["cpuPercent"] = p.cpu_percent() + memory = dict() + memory["rss"] = mem.rss + memory["vms"] = mem.vms + memory["shared"] = mem.shared + memory["text"] = mem.text + memory["lib"] = mem.lib + memory["data"] = mem.data + memory["dirty"] = mem.dirty + result["memory"] = memory + # io = p.io_counters() # Permission denied: '/proc/{pid}/io' + return result +# end define + + +def Telemetry(local, ton): + sendTelemetry = local.db.get("sendTelemetry") + if sendTelemetry is not True: + return + # end if + + # Get validator status + data = dict() + data["adnlAddr"] = ton.GetAdnlAddr() + data["validatorStatus"] = ton.GetValidatorStatus() + data["cpuNumber"] = psutil.cpu_count() + data["cpuLoad"] = get_load_avg() + data["netLoad"] = ton.GetStatistics("netLoadAvg") + data["tps"] = ton.GetStatistics("tpsAvg") + data["disksLoad"] = ton.GetStatistics("disksLoadAvg") + data["disksLoadPercent"] = ton.GetStatistics("disksLoadPercentAvg") + data["iops"] = ton.GetStatistics("iopsAvg") + data["pps"] = ton.GetStatistics("ppsAvg") + data["dbUsage"] = ton.GetDbUsage() + data["memory"] = GetMemoryInfo() + data["swap"] = GetSwapInfo() + data["uname"] = GetUname() + data["vprocess"] = GetValidatorProcessInfo() + elections = local.try_function(ton.GetElectionEntries) + complaints = local.try_function(ton.GetComplaints) + + # Get git hashes + gitHashes = dict() + gitHashes["mytonctrl"] = get_git_hash("/usr/src/mytonctrl") + gitHashes["validator"] = GetBinGitHash( + "/usr/bin/ton/validator-engine/validator-engine") + data["gitHashes"] = gitHashes + data["stake"] = local.db.get("stake") + + # Get validator config + vconfig = ton.GetValidatorConfig() + data["fullnode_adnl"] = vconfig.fullnode + + # Send data to toncenter server + liteUrl_default = "https://telemetry.toncenter.com/report_status" + liteUrl = local.db.get("telemetryLiteUrl", liteUrl_default) + output = json.dumps(data) + resp = requests.post(liteUrl, data=output, timeout=3) +# end define + + +def GetBinGitHash(path, short=False): + if not os.path.isfile(path): + return + args = [path, "--version"] + process = subprocess.run(args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3) + output = process.stdout.decode("utf-8") + if "build information" not in output: + return + buff = output.split(' ') + start = buff.index("Commit:") + 1 + result = buff[start].replace(',', '') + if short is True: + result = result[:7] + return result +# end define + + +def OverlayTelemetry(local, ton): + sendTelemetry = local.db.get("sendTelemetry") + if sendTelemetry is not True: + return + # end if + + # Get validator status + data = dict() + data["adnlAddr"] = ton.GetAdnlAddr() + data["overlaysStats"] = ton.GetOverlaysStats() + + # Send data to toncenter server + overlayUrl_default = "https://telemetry.toncenter.com/report_overlays" + overlayUrl = local.db.get("overlayTelemetryUrl", overlayUrl_default) + output = json.dumps(data) + resp = requests.post(overlayUrl, data=output, timeout=3) +# end define + + +def Complaints(local, ton): + validatorIndex = ton.GetValidatorIndex() + if validatorIndex < 0: + return + # end if + + # Voting for complaints + config32 = ton.GetConfig32() + electionId = config32.get("startWorkTime") + complaintsHashes = ton.SaveComplaints(electionId) + complaints = ton.GetComplaints(electionId) + for key, item in complaints.items(): + complaintHash = item.get("hash") + complaintHash_hex = Dec2HexAddr(complaintHash) + if complaintHash_hex in complaintsHashes: + ton.VoteComplaint(electionId, complaintHash) +# end define + + +def Slashing(local, ton): + isSlashing = local.db.get("isSlashing") + if isSlashing is not True: + return + # end if + + # Creating complaints + slash_time = local.buffer.slash_time + config32 = ton.GetConfig32() + start = config32.get("startWorkTime") + end = config32.get("endWorkTime") + local.add_log("slash_time {}, start {}, end {}".format(slash_time, start, end), "debug") + if slash_time != start: + end -= 60 + ton.CheckValidators(start, end) + local.buffer.slash_time = start +# end define + + +def ScanLiteServers(local, ton): + # Считать список серверов + filePath = ton.liteClient.configPath + file = open(filePath, 'rt') + text = file.read() + file.close() + data = json.loads(text) + + # Пройтись по серверам + result = list() + liteservers = data.get("liteservers") + for index in range(len(liteservers)): + try: + ton.liteClient.Run("last", index=index) + result.append(index) + except: + pass + # end for + + # Записать данные в базу + local.db["liteServers"] = result +# end define + + +def General(local): + local.add_log("start General function", "debug") + ton = MyTonCore(local) + scanner = Dict() + # scanner.Run() + + # Запустить потоки + local.start_cycle(Elections, sec=600, args=(local, ton, )) + local.start_cycle(Statistics, sec=10, args=(local, )) + local.start_cycle(Offers, sec=600, args=(local, ton, )) + local.start_cycle(Complaints, sec=600, args=(local, ton, )) + local.start_cycle(Slashing, sec=600, args=(local, ton, )) + local.start_cycle(Domains, sec=600, args=(local, ton, )) + local.start_cycle(Telemetry, sec=60, args=(local, ton, )) + local.start_cycle(OverlayTelemetry, sec=7200, args=(local, ton, )) + local.start_cycle(ScanLiteServers, sec=60, args=(local, ton,)) + thr_sleep() +# end define + + +def mytoncore(): + from mypylib.mypylib import MyPyClass + + local = MyPyClass('mytoncore.py') + print('Local DB path:', local.buffer['localdbFileName']) + Init(local) + General(local) diff --git a/mytoncore/liteclient.py b/mytoncore/liteclient.py new file mode 100644 index 00000000..8a7a25df --- /dev/null +++ b/mytoncore/liteclient.py @@ -0,0 +1,44 @@ +import random +import subprocess + + +class LiteClient: + def __init__(self, local): + self.local = local + self.appPath = None + self.configPath = None + self.pubkeyPath = None + self.addr = None + self.ton = None # magic + #end define + + def Run(self, cmd, **kwargs): + index = kwargs.get("index") + liteclient_timeout = self.local.db.liteclient_timeout if self.local.db.liteclient_timeout else 3 + timeout = kwargs.get("timeout", liteclient_timeout) + useLocalLiteServer = kwargs.get("useLocalLiteServer", True) + validatorStatus = self.ton.GetValidatorStatus() + validatorOutOfSync = validatorStatus.get("outOfSync") + args = [self.appPath, "--global-config", self.configPath, "--verbosity", "0", "--cmd", cmd] + if index is not None: + index = str(index) + args += ["-i", index] + elif useLocalLiteServer and self.pubkeyPath and validatorOutOfSync < 20: + args = [self.appPath, "--addr", self.addr, "--pub", self.pubkeyPath, "--verbosity", "0", "--cmd", cmd] + else: + liteServers = self.local.db.get("liteServers") + if liteServers is not None and len(liteServers): + index = random.choice(liteServers) + index = str(index) + args += ["-i", index] + #end if + + process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + output = process.stdout.decode("utf-8") + err = process.stderr.decode("utf-8") + if len(err) > 0: + self.local.add_log("args: {args}".format(args=args), "error") + raise Exception("LiteClient error: {err}".format(err=err)) + return output + #end define +#end class diff --git a/mytoncore/models.py b/mytoncore/models.py new file mode 100644 index 00000000..474fa276 --- /dev/null +++ b/mytoncore/models.py @@ -0,0 +1,177 @@ +import os + + +class Wallet: + def __init__(self, name, path, version): + self.name = name + self.path = path + self.addrFilePath = f"{path}.addr" + self.privFilePath = f"{path}.pk" + self.bocFilePath = f"{path}-query.boc" + self.addrFull = None + self.workchain = None + self.addr = None + self.addrB64 = None + self.addrB64_init = None + self.oldseqno = None + self.account = None + self.subwallet = None + self.version = version + #end define + + def Delete(self): + os.remove(self.addrFilePath) + os.remove(self.privFilePath) + #end define +#end class + + +class Account: + def __init__(self, workchain, addr): + self.workchain = workchain + self.addr = addr + self.addrB64 = None + self.addrFull = None + self.status = "empty" + self.balance = 0 + self.lt = None + self.hash = None + self.codeHash = None + #end define +#end class + + +class Domain(dict): + def __init__(self): + self["name"] = None + self["adnlAddr"] = None + self["walletName"] = None + #end define +#end class + + +class Block(): + def __init__(self, str=None): + self.workchain = None + self.shardchain = None + self.seqno = None + self.rootHash = None + self.fileHash = None + self.ParsBlock(str) + #end define + + def ParsBlock(self, str): + if str is None: + return + buff = str.split(':') + self.rootHash = buff[1] + self.fileHash = buff[2] + buff = buff[0] + buff = buff.replace('(', '') + buff = buff.replace(')', '') + buff = buff.split(',') + self.workchain = int(buff[0]) + self.shardchain = buff[1] + self.seqno = int(buff[2]) + #end define + + def __str__(self): + result = f"({self.workchain},{self.shardchain},{self.seqno}):{self.rootHash}:{self.fileHash}" + return result + #end define + + def __repr__(self): + return self.__str__() + #end define + + def __eq__(self, other): + if other is None: + return False + return self.rootHash == other.rootHash and self.fileHash == other.fileHash + #end define +#end class + + +class Trans(): + def __init__(self, block, addr=None, lt=None, hash=None): + self.block = block + self.addr = addr + self.lt = lt + self.hash = hash + #end define + + def __str__(self): + return str(self.__dict__) + #end define + + def __repr__(self): + return self.__str__() + #end define + + def __eq__(self, other): + if other is None: + return False + return self.hash == other.hash + #end define +#end class + + +class Message(): + def __init__(self): + self.trans = None + self.type = None + self.time = None + self.srcWorkchain = None + self.destWorkchain = None + self.srcAddr = None + self.destAddr = None + self.value = None + self.body = None + self.comment = None + self.ihr_fee = None + self.fwd_fee = None + self.total_fees = None + self.ihr_disabled = None + self.hash = None + #end define + + def GetFullAddr(self, workchain, addr): + if addr is None: + return + return f"{workchain}:{addr}" + #end define + + def __str__(self): + return str(self.__dict__) + #end define + + def __repr__(self): + return self.__str__() + #end define + + def __eq__(self, other): + if other is None: + return False + return self.hash == other.hash + #end define +#end class + + +class Pool: + def __init__(self, name, path): + self.name = name + self.path = path + self.addrFilePath = f"{path}.addr" + self.bocFilePath = f"{path}-query.boc" + self.addrFull = None + self.workchain = None + self.addr = None + self.addrB64 = None + self.addrB64_init = None + self.account = None + #end define + + def Delete(self): + os.remove(self.addrFilePath) + #end define +#end class diff --git a/mytoncore.py b/mytoncore/mytoncore.py old mode 100755 new mode 100644 similarity index 75% rename from mytoncore.py rename to mytoncore/mytoncore.py index fd1a9671..e912adf3 --- a/mytoncore.py +++ b/mytoncore/mytoncore.py @@ -1,274 +1,42 @@ -#!/usr/bin/env python3 -# -*- coding: utf_8 -*-l - -from fastcrc import crc16 -import struct -import random -import hashlib -import requests +import os +import base64 +import time import re -from mypylib.mypylib import * - -local = MyPyClass(__file__) - -class LiteClient: - def __init__(self): - self.appPath = None - self.configPath = None - self.pubkeyPath = None - self.addr = None - self.ton = None # magic - #end define - - def Run(self, cmd, **kwargs): - index = kwargs.get("index") - liteclient_timeout = local.db.liteclient_timeout if local.db.liteclient_timeout else 3 - timeout = kwargs.get("timeout", liteclient_timeout) - useLocalLiteServer = kwargs.get("useLocalLiteServer", True) - validatorStatus = self.ton.GetValidatorStatus() - validatorOutOfSync = validatorStatus.get("outOfSync") - args = [self.appPath, "--global-config", self.configPath, "--verbosity", "0", "--cmd", cmd] - if index is not None: - index = str(index) - args += ["-i", index] - elif useLocalLiteServer and self.pubkeyPath and validatorOutOfSync < 20: - args = [self.appPath, "--addr", self.addr, "--pub", self.pubkeyPath, "--verbosity", "0", "--cmd", cmd] - else: - liteServers = local.db.get("liteServers") - if liteServers is not None and len(liteServers): - index = random.choice(liteServers) - index = str(index) - args += ["-i", index] - #end if - - process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - output = process.stdout.decode("utf-8") - err = process.stderr.decode("utf-8") - if len(err) > 0: - local.add_log("args: {args}".format(args=args), "error") - raise Exception("LiteClient error: {err}".format(err=err)) - return output - #end define -#end class - -class ValidatorConsole: - def __init__(self): - self.appPath = None - self.privKeyPath = None - self.pubKeyPath = None - self.addr = None - #end define - - def Run(self, cmd, **kwargs): - console_timeout = local.db.console_timeout if local.db.console_timeout else 3 - timeout = kwargs.get("timeout", console_timeout) - if self.appPath is None or self.privKeyPath is None or self.pubKeyPath is None: - raise Exception("ValidatorConsole error: Validator console is not settings") - args = [self.appPath, "-k", self.privKeyPath, "-p", self.pubKeyPath, "-a", self.addr, "-v", "0", "--cmd", cmd] - process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - output = process.stdout.decode("utf-8") - err = process.stderr.decode("utf-8") - if len(err) > 0: - local.add_log("args: {args}".format(args=args), "error") - raise Exception("ValidatorConsole error: {err}".format(err=err)) - return output - #end define -#end class - -class Fift: - def __init__(self): - self.appPath = None - self.libsPath = None - self.smartcontsPath = None - #end define - - def Run(self, args, **kwargs): - fift_timeout = local.db.fift_timeout if local.db.fift_timeout else 3 - timeout = kwargs.get("timeout", fift_timeout) - for i in range(len(args)): - args[i] = str(args[i]) - includePath = self.libsPath + ':' + self.smartcontsPath - args = [self.appPath, "-I", includePath, "-s"] + args - process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) - output = process.stdout.decode("utf-8") - err = process.stderr.decode("utf-8") - if len(err) > 0: - local.add_log("args: {args}".format(args=args), "error") - raise Exception("Fift error: {err}".format(err=err)) - return output - #end define -#end class - -class Wallet: - def __init__(self, name, path, version): - self.name = name - self.path = path - self.addrFilePath = f"{path}.addr" - self.privFilePath = f"{path}.pk" - self.bocFilePath = f"{path}-query.boc" - self.addrFull = None - self.workchain = None - self.addr = None - self.addrB64 = None - self.addrB64_init = None - self.oldseqno = None - self.account = None - self.subwallet = None - self.version = version - #end define - - def Delete(self): - os.remove(self.addrFilePath) - os.remove(self.privFilePath) - #end define -#end class - -class Account: - def __init__(self, workchain, addr): - self.workchain = workchain - self.addr = addr - self.addrB64 = None - self.addrFull = None - self.status = "empty" - self.balance = 0 - self.lt = None - self.hash = None - self.codeHash = None - #end define -#end class - -class Domain(dict): - def __init__(self): - self["name"] = None - self["adnlAddr"] = None - self["walletName"] = None - #end define -#end class - -class Block(): - def __init__(self, str=None): - self.workchain = None - self.shardchain = None - self.seqno = None - self.rootHash = None - self.fileHash = None - self.ParsBlock(str) - #end define - - def ParsBlock(self, str): - if str is None: - return - buff = str.split(':') - self.rootHash = buff[1] - self.fileHash = buff[2] - buff = buff[0] - buff = buff.replace('(', '') - buff = buff.replace(')', '') - buff = buff.split(',') - self.workchain = int(buff[0]) - self.shardchain = buff[1] - self.seqno = int(buff[2]) - #end define - - def __str__(self): - result = f"({self.workchain},{self.shardchain},{self.seqno}):{self.rootHash}:{self.fileHash}" - return result - #end define - - def __repr__(self): - return self.__str__() - #end define - - def __eq__(self, other): - if other is None: - return False - return self.rootHash == other.rootHash and self.fileHash == other.fileHash - #end define -#end class - -class Trans(): - def __init__(self, block, addr=None, lt=None, hash=None): - self.block = block - self.addr = addr - self.lt = lt - self.hash = hash - #end define - - def __str__(self): - return str(self.__dict__) - #end define - - def __repr__(self): - return self.__str__() - #end define - - def __eq__(self, other): - if other is None: - return False - return self.hash == other.hash - #end define -#end class - -class Message(): - def __init__(self): - self.trans = None - self.type = None - self.time = None - self.srcWorkchain = None - self.destWorkchain = None - self.srcAddr = None - self.destAddr = None - self.value = None - self.body = None - self.comment = None - self.ihr_fee = None - self.fwd_fee = None - self.total_fees = None - self.ihr_disabled = None - #end define - - def GetFullAddr(self, workchain, addr): - if addr is None: - return - return f"{workchain}:{addr}" - #end define - - def __str__(self): - return str(self.__dict__) - #end define +import json +import hashlib +import struct +import psutil +import crc16 # TODO: check this library! +import fastcrc +import subprocess - def __repr__(self): - return self.__str__() - #end define +from fastcrc import crc16 - def __eq__(self, other): - if other is None: - return False - return self.hash == other.hash - #end define -#end class +from mytoncore.utils import xhex2hex, ng2g +from mytoncore.liteclient import LiteClient +from mytoncore.validator_console import ValidatorConsole +from mytoncore.fift import Fift +from mytoncore.models import ( + Wallet, + Account, + Domain, + Block, + Trans, + Message, + Pool, +) + +from mypylib.mypylib import ( + parse, + get_timestamp, + timestamp2datetime, + dec2hex, +) -class Pool: - def __init__(self, name, path): - self.name = name - self.path = path - self.addrFilePath = f"{path}.addr" - self.bocFilePath = f"{path}-query.boc" - self.addrFull = None - self.workchain = None - self.addr = None - self.addrB64 = None - self.addrB64_init = None - self.account = None - #end define - - def Delete(self): - os.remove(self.addrFilePath) - #end define -#end class class MyTonCore(): - def __init__(self): + def __init__(self, local): + self.local = local self.walletsDir = None self.dbFile = None self.contractsDir = None @@ -276,9 +44,9 @@ def __init__(self): self.tempDir = None self.nodeName = None - self.liteClient = LiteClient() - self.validatorConsole = ValidatorConsole() - self.fift = Fift() + self.liteClient = LiteClient(self.local) + self.validatorConsole = ValidatorConsole(self.local) + self.fift = Fift(self.local) self.Refresh() self.Init() @@ -293,22 +61,21 @@ def Init(self): def Refresh(self): if self.dbFile: - local.load_db(self.dbFile) - #end if + self.local.load_db(self.dbFile) if not self.walletsDir: - self.walletsDir = local.buffer.my_work_dir + "wallets/" - self.contractsDir = local.buffer.my_work_dir + "contracts/" - self.poolsDir = local.buffer.my_work_dir + "pools/" - self.tempDir = local.buffer.my_temp_dir + self.walletsDir = self.local.buffer.my_work_dir + "wallets/" + self.contractsDir = self.local.buffer.my_work_dir + "contracts/" + self.poolsDir = self.local.buffer.my_work_dir + "pools/" + self.tempDir = self.local.buffer.my_work_dir - self.nodeName = local.db.get("nodeName") + self.nodeName = self.local.db.get("nodeName") if self.nodeName is None: self.nodeName="" else: self.nodeName = self.nodeName + "_" - liteClient = local.db.get("liteClient") + liteClient = self.local.db.get("liteClient") if liteClient is not None: self.liteClient.ton = self # magic self.liteClient.appPath = liteClient["appPath"] @@ -319,7 +86,7 @@ def Refresh(self): self.liteClient.addr = "{0}:{1}".format(liteServer["ip"], liteServer["port"]) #end if - validatorConsole = local.db.get("validatorConsole") + validatorConsole = self.local.db.get("validatorConsole") if validatorConsole is not None: self.validatorConsole.appPath = validatorConsole["appPath"] self.validatorConsole.privKeyPath = validatorConsole["privKeyPath"] @@ -327,7 +94,7 @@ def Refresh(self): self.validatorConsole.addr = validatorConsole["addr"] #end if - fift = local.db.get("fift") + fift = self.local.db.get("fift") if fift is not None: self.fift.appPath = fift["appPath"] self.fift.libsPath = fift["libsPath"] @@ -339,18 +106,18 @@ def Refresh(self): #end define def CheckConfigFile(self, fift, liteClient): - mconfig_path = local.buffer.db_path + mconfig_path = self.local.buffer.db_path backup_path = mconfig_path + ".backup" if fift is None or liteClient is None: - local.add_log("The config file is broken", "warning") - print(f"local.db: {local.db}") + self.local.add_log("The config file is broken", "warning") + print(f"self.local.db: {self.local.db}") if os.path.isfile(backup_path): - local.add_log("Restoring the configuration file", "info") + self.local.add_log("Restoring the configuration file", "info") args = ["cp", backup_path, mconfig_path] subprocess.run(args) self.Refresh() elif os.path.isfile(backup_path) == False: - local.add_log("Create backup config file", "info") + self.local.add_log("Create backup config file", "info") args = ["cp", mconfig_path, backup_path] subprocess.run(args) #end define @@ -387,7 +154,7 @@ def GetVarFromWorkerOutput(self, text, search): #end define def GetSeqno(self, wallet): - local.add_log("start GetSeqno function", "debug") + self.local.add_log("start GetSeqno function", "debug") cmd = "runmethodfull {addr} seqno".format(addr=wallet.addrB64) result = self.liteClient.Run(cmd) if "cannot run any methods" in result: @@ -402,7 +169,7 @@ def GetSeqno(self, wallet): #end define def GetAccount(self, inputAddr): - #local.add_log("start GetAccount function", "debug") + #self.local.add_log("start GetAccount function", "debug") workchain, addr = self.ParseInputAddr(inputAddr) account = Account(workchain, addr) cmd = "getaccount {inputAddr}".format(inputAddr=inputAddr) @@ -447,7 +214,7 @@ def GetCodeHash(self, code): #end define def GetAccountHistory(self, account, limit): - local.add_log("start GetAccountHistory function", "debug") + self.local.add_log("start GetAccountHistory function", "debug") addr = f"{account.workchain}:{account.addr}" lt = account.lt transHash = account.hash @@ -636,7 +403,7 @@ def GetDomainAddr(self, domainName): #end define def GetDomainEndTime(self, domainName): - local.add_log("start GetDomainEndTime function", "debug") + self.local.add_log("start GetDomainEndTime function", "debug") buff = domainName.split('.') subdomain = buff.pop(0) dnsDomain = ".".join(buff) @@ -652,7 +419,7 @@ def GetDomainEndTime(self, domainName): #end define def GetDomainAdnlAddr(self, domainName): - local.add_log("start GetDomainAdnlAddr function", "debug") + self.local.add_log("start GetDomainAdnlAddr function", "debug") cmd = "dnsresolve {domainName} 1".format(domainName=domainName) result = self.liteClient.Run(cmd) lines = result.split('\n') @@ -665,7 +432,7 @@ def GetDomainAdnlAddr(self, domainName): #end define def GetLocalWallet(self, walletName, version=None, subwallet=None): - local.add_log("start GetLocalWallet function", "debug") + self.local.add_log("start GetLocalWallet function", "debug") if walletName is None: return None walletPath = self.walletsDir + walletName @@ -677,7 +444,7 @@ def GetLocalWallet(self, walletName, version=None, subwallet=None): #end define def GetWalletFromFile(self, filePath, version): - local.add_log("start GetWalletFromFile function", "debug") + self.local.add_log("start GetWalletFromFile function", "debug") # Check input args if (".addr" in filePath): filePath = filePath.replace(".addr", '') @@ -696,7 +463,7 @@ def GetWalletFromFile(self, filePath, version): #end define def GetHighWalletFromFile(self, filePath, subwallet, version): - local.add_log("start GetHighWalletFromFile function", "debug") + self.local.add_log("start GetHighWalletFromFile function", "debug") # Check input args if (".addr" in filePath): filePath = filePath.replace(".addr", '') @@ -728,7 +495,7 @@ def AddrFile2Object(self, object): #end define def WalletVersion2Wallet(self, wallet): - local.add_log("start WalletVersion2Wallet function", "debug") + self.local.add_log("start WalletVersion2Wallet function", "debug") if wallet.version is not None: return walletsVersionList = self.GetWalletsVersionList() @@ -737,7 +504,7 @@ def WalletVersion2Wallet(self, wallet): if version is None: version = self.GetWalletVersionFromHash(account.codeHash) if version is None: - local.add_log("Wallet version not found: " + wallet.addrB64, "warning") + self.local.add_log("Wallet version not found: " + wallet.addrB64, "warning") return #end if @@ -748,11 +515,11 @@ def WalletVersion2Wallet(self, wallet): def SetWalletVersion(self, addrB64, version): walletsVersionList = self.GetWalletsVersionList() walletsVersionList[addrB64] = version - local.save() + self.local.save() #end define def GetWalletVersionFromHash(self, inputHash): - local.add_log("start GetWalletVersionFromHash function", "debug") + self.local.add_log("start GetWalletVersionFromHash function", "debug") arr = dict() arr["v1r1"] = "d670136510daff4fee1889b8872c4c1e89872ffa1fe58a23a5f5d99cef8edf32" arr["v1r2"] = "2705a31a7ac162295c8aed0761cc6e031ab65521dd7b4a14631099e02de99e18" @@ -771,10 +538,10 @@ def GetWalletVersionFromHash(self, inputHash): def GetWalletsVersionList(self): bname = "walletsVersionList" - walletsVersionList = local.db.get(bname) + walletsVersionList = self.local.db.get(bname) if walletsVersionList is None: walletsVersionList = dict() - local.db[bname] = walletsVersionList + self.local.db[bname] = walletsVersionList return walletsVersionList #end define @@ -786,7 +553,7 @@ def GetFullConfigAddr(self): return buff #end if - local.add_log("start GetFullConfigAddr function", "debug") + self.local.add_log("start GetFullConfigAddr function", "debug") result = self.liteClient.Run("getconfig 0") configAddr_hex = self.GetVarFromWorkerOutput(result, "config_addr:x") fullConfigAddr = "-1:{configAddr_hex}".format(configAddr_hex=configAddr_hex) @@ -805,7 +572,7 @@ def GetFullElectorAddr(self): #end if # Get data - local.add_log("start GetFullElectorAddr function", "debug") + self.local.add_log("start GetFullElectorAddr function", "debug") result = self.liteClient.Run("getconfig 1") electorAddr_hex = self.GetVarFromWorkerOutput(result, "elector_addr:x") fullElectorAddr = "-1:{electorAddr_hex}".format(electorAddr_hex=electorAddr_hex) @@ -823,7 +590,7 @@ def GetFullMinterAddr(self): return buff #end if - local.add_log("start GetFullMinterAddr function", "debug") + self.local.add_log("start GetFullMinterAddr function", "debug") result = self.liteClient.Run("getconfig 2") minterAddr_hex = self.GetVarFromWorkerOutput(result, "minter_addr:x") fullMinterAddr = "-1:{minterAddr_hex}".format(minterAddr_hex=minterAddr_hex) @@ -841,7 +608,7 @@ def GetFullDnsRootAddr(self): return buff #end if - local.add_log("start GetFullDnsRootAddr function", "debug") + self.local.add_log("start GetFullDnsRootAddr function", "debug") result = self.liteClient.Run("getconfig 4") dnsRootAddr_hex = self.GetVarFromWorkerOutput(result, "dns_root_addr:x") fullDnsRootAddr = "-1:{dnsRootAddr_hex}".format(dnsRootAddr_hex=dnsRootAddr_hex) @@ -859,7 +626,7 @@ def GetActiveElectionId(self, fullElectorAddr): return buff #end if - local.add_log("start GetActiveElectionId function", "debug") + self.local.add_log("start GetActiveElectionId function", "debug") cmd = "runmethodfull {fullElectorAddr} active_election_id".format(fullElectorAddr=fullElectorAddr) result = self.liteClient.Run(cmd) activeElectionId = self.GetVarFromWorkerOutput(result, "result") @@ -873,26 +640,26 @@ def GetActiveElectionId(self, fullElectorAddr): #end define def GetValidatorsElectedFor(self): - local.add_log("start GetValidatorsElectedFor function", "debug") + self.local.add_log("start GetValidatorsElectedFor function", "debug") config15 = self.GetConfig15() return config15["validatorsElectedFor"] #end define def GetMinStake(self): - local.add_log("start GetMinStake function", "debug") + self.local.add_log("start GetMinStake function", "debug") config17 = self.GetConfig17() return config17["minStake"] #end define def GetRootWorkchainEnabledTime(self): - local.add_log("start GetRootWorkchainEnabledTime function", "debug") + self.local.add_log("start GetRootWorkchainEnabledTime function", "debug") config12 = self.GetConfig(12) enabledTime = config12["workchains"]["root"]["node"]["value"]["enabled_since"] return enabledTime #end define def GetTotalValidators(self): - local.add_log("start GetTotalValidators function", "debug") + self.local.add_log("start GetTotalValidators function", "debug") config34 = self.GetConfig34() result = config34["totalValidators"] return result @@ -1036,7 +803,7 @@ def GetValidatorStatus(self): return buff #end if - # local.add_log("start GetValidatorStatus function", "debug") + self.local.add_log("start GetValidatorStatus function", "debug") validatorStatus = dict() try: validatorStatus["isWorking"] = True @@ -1053,11 +820,11 @@ def GetValidatorStatus(self): validatorStatus["keymasterchainblock"] = self.GVS_GetItemFromBuff(buff) buff = parse(result, "rotatemasterchainblock", '\n') validatorStatus["rotatemasterchainblock"] = self.GVS_GetItemFromBuff(buff) - validatorStatus["transNum"] = local.buffer.get("transNum", -1) - validatorStatus["blocksNum"] = local.buffer.get("blocksNum", -1) - validatorStatus["masterBlocksNum"] = local.buffer.get("masterBlocksNum", -1) + validatorStatus["transNum"] = self.local.buffer.get("transNum", -1) + validatorStatus["blocksNum"] = self.local.buffer.get("blocksNum", -1) + validatorStatus["masterBlocksNum"] = self.local.buffer.get("masterBlocksNum", -1) except Exception as ex: - local.add_log(f"GetValidatorStatus warning: {ex}", "warning") + self.local.add_log(f"GetValidatorStatus warning: {ex}", "warning") validatorStatus["isWorking"] = False validatorStatus["unixtime"] = get_timestamp() validatorStatus["masterchainblocktime"] = 0 @@ -1089,7 +856,7 @@ def GetConfig(self, configId): #end if text = "start GetConfig function ({})".format(configId) - local.add_log(text, "debug") + self.local.add_log(text, "debug") cmd = "getconfig {configId}".format(configId=configId) result = self.liteClient.Run(cmd) start = result.find("ConfigParam") @@ -1128,7 +895,7 @@ def GetConfig32(self): return buff #end if - local.add_log("start GetConfig32 function", "debug") + self.local.add_log("start GetConfig32 function", "debug") config32 = dict() result = self.liteClient.Run("getconfig 32") config32["totalValidators"] = int(parse(result, "total:", ' ')) @@ -1164,7 +931,7 @@ def GetConfig34(self): return buff #end if - local.add_log("start GetConfig34 function", "debug") + self.local.add_log("start GetConfig34 function", "debug") config34 = dict() result = self.liteClient.Run("getconfig 34") config34["totalValidators"] = int(parse(result, "total:", ' ')) @@ -1201,7 +968,7 @@ def GetConfig36(self): return buff #end if - local.add_log("start GetConfig36 function", "debug") + self.local.add_log("start GetConfig36 function", "debug") config36 = dict() try: result = self.liteClient.Run("getconfig 36") @@ -1231,21 +998,21 @@ def GetConfig36(self): #end define def CreateNewKey(self): - local.add_log("start CreateNewKey function", "debug") + self.local.add_log("start CreateNewKey function", "debug") result = self.validatorConsole.Run("newkey") key = parse(result, "created new key ", '\n') return key #end define def GetPubKeyBase64(self, key): - local.add_log("start GetPubKeyBase64 function", "debug") + self.local.add_log("start GetPubKeyBase64 function", "debug") result = self.validatorConsole.Run("exportpub " + key) validatorPubkey_b64 = parse(result, "got public key: ", '\n') return validatorPubkey_b64 #end define def GetPubKey(self, key): - local.add_log("start GetPubKey function", "debug") + self.local.add_log("start GetPubKey function", "debug") pubkey_b64 = self.GetPubKeyBase64(key) buff = pubkey_b64.encode("utf-8") buff = base64.b64decode(buff) @@ -1256,7 +1023,7 @@ def GetPubKey(self, key): #end define def AddKeyToValidator(self, key, startWorkTime, endWorkTime): - local.add_log("start AddKeyToValidator function", "debug") + self.local.add_log("start AddKeyToValidator function", "debug") output = False cmd = "addpermkey {key} {startWorkTime} {endWorkTime}".format(key=key, startWorkTime=startWorkTime, endWorkTime=endWorkTime) result = self.validatorConsole.Run(cmd) @@ -1266,7 +1033,7 @@ def AddKeyToValidator(self, key, startWorkTime, endWorkTime): #end define def AddKeyToTemp(self, key, endWorkTime): - local.add_log("start AddKeyToTemp function", "debug") + self.local.add_log("start AddKeyToTemp function", "debug") output = False result = self.validatorConsole.Run("addtempkey {key} {key} {endWorkTime}".format(key=key, endWorkTime=endWorkTime)) if ("success" in result): @@ -1275,7 +1042,7 @@ def AddKeyToTemp(self, key, endWorkTime): #end define def AddAdnlAddrToValidator(self, adnlAddr): - local.add_log("start AddAdnlAddrToValidator function", "debug") + self.local.add_log("start AddAdnlAddrToValidator function", "debug") output = False result = self.validatorConsole.Run("addadnl {adnlAddr} 0".format(adnlAddr=adnlAddr)) if ("success" in result): @@ -1284,12 +1051,12 @@ def AddAdnlAddrToValidator(self, adnlAddr): #end define def GetAdnlAddr(self): - adnlAddr = local.db.get("adnlAddr") + adnlAddr = self.local.db.get("adnlAddr") return adnlAddr #end define def AttachAdnlAddrToValidator(self, adnlAddr, key, endWorkTime): - local.add_log("start AttachAdnlAddrToValidator function", "debug") + self.local.add_log("start AttachAdnlAddrToValidator function", "debug") output = False result = self.validatorConsole.Run("addvalidatoraddr {key} {adnlAddr} {endWorkTime}".format(adnlAddr=adnlAddr, key=key, endWorkTime=endWorkTime)) if ("success" in result): @@ -1298,7 +1065,7 @@ def AttachAdnlAddrToValidator(self, adnlAddr, key, endWorkTime): #end define def CreateConfigProposalRequest(self, offerHash, validatorIndex): - local.add_log("start CreateConfigProposalRequest function", "debug") + self.local.add_log("start CreateConfigProposalRequest function", "debug") fileName = self.tempDir + self.nodeName + "proposal_validator-to-sign.req" args = ["config-proposal-vote-req.fif", "-i", validatorIndex, offerHash, fileName] result = self.fift.Run(args) @@ -1316,7 +1083,7 @@ def CreateConfigProposalRequest(self, offerHash, validatorIndex): #end define def CreateComplaintRequest(self, electionId , complaintHash, validatorIndex): - local.add_log("start CreateComplaintRequest function", "debug") + self.local.add_log("start CreateComplaintRequest function", "debug") fileName = self.tempDir + "complaint_validator-to-sign.req" args = ["complaint-vote-req.fif", validatorIndex, electionId, complaintHash, fileName] result = self.fift.Run(args) @@ -1334,7 +1101,7 @@ def CreateComplaintRequest(self, electionId , complaintHash, validatorIndex): #end define def PrepareComplaint(self, electionId, inputFileName): - local.add_log("start PrepareComplaint function", "debug") + self.local.add_log("start PrepareComplaint function", "debug") fileName = self.tempDir + "complaint-msg-body.boc" args = ["envelope-complaint.fif", electionId, inputFileName, fileName] result = self.fift.Run(args) @@ -1343,7 +1110,7 @@ def PrepareComplaint(self, electionId, inputFileName): #end define def CreateElectionRequest(self, wallet, startWorkTime, adnlAddr, maxFactor): - local.add_log("start CreateElectionRequest function", "debug") + self.local.add_log("start CreateElectionRequest function", "debug") fileName = self.tempDir + self.nodeName + str(startWorkTime) + "_validator-to-sign.bin" args = ["validator-elect-req.fif", wallet.addrB64, startWorkTime, maxFactor, adnlAddr, fileName] result = self.fift.Run(args) @@ -1361,7 +1128,7 @@ def CreateElectionRequest(self, wallet, startWorkTime, adnlAddr, maxFactor): #end define def GetValidatorSignature(self, validatorKey, var1): - local.add_log("start GetValidatorSignature function", "debug") + self.local.add_log("start GetValidatorSignature function", "debug") cmd = "sign {validatorKey} {var1}".format(validatorKey=validatorKey, var1=var1) result = self.validatorConsole.Run(cmd) validatorSignature = parse(result, "got signature ", '\n') @@ -1369,7 +1136,7 @@ def GetValidatorSignature(self, validatorKey, var1): #end define def SignElectionRequestWithValidator(self, wallet, startWorkTime, adnlAddr, validatorPubkey_b64, validatorSignature, maxFactor): - local.add_log("start SignElectionRequestWithValidator function", "debug") + self.local.add_log("start SignElectionRequestWithValidator function", "debug") fileName = self.tempDir + self.nodeName + str(startWorkTime) + "_validator-query.boc" args = ["validator-elect-signed.fif", wallet.addrB64, startWorkTime, maxFactor, adnlAddr, validatorPubkey_b64, validatorSignature, fileName] result = self.fift.Run(args) @@ -1379,7 +1146,7 @@ def SignElectionRequestWithValidator(self, wallet, startWorkTime, adnlAddr, vali #end define def SignBocWithWallet(self, wallet, bocPath, dest, coins, **kwargs): - local.add_log("start SignBocWithWallet function", "debug") + self.local.add_log("start SignBocWithWallet function", "debug") flags = kwargs.get("flags", list()) subwalletDefault = 698983191 + wallet.workchain # 0x29A9A317 + workchain subwallet = kwargs.get("subwallet", subwalletDefault) @@ -1396,7 +1163,7 @@ def SignBocWithWallet(self, wallet, bocPath, dest, coins, **kwargs): if bounceable == False and destAccount.status == "active": flags += ["-b"] text = "Find non-bounceable flag, but destination account already active. Using bounceable flag" - local.add_log(text, "warning") + self.local.AddLog(text, "warning") elif "-n" not in flags and bounceable == True and destAccount.status != "active": raise Exception("Find bounceable flag, but destination account is not active. Use non-bounceable address or flag -n") #end if @@ -1420,10 +1187,10 @@ def SignBocWithWallet(self, wallet, bocPath, dest, coins, **kwargs): #end define def SendFile(self, filePath, wallet=None, **kwargs): - local.add_log("start SendFile function: " + filePath, "debug") + self.local.add_log("start SendFile function: " + filePath, "debug") timeout = kwargs.get("timeout", 30) remove = kwargs.get("remove", True) - duplicateSendfile = local.db.get("duplicateSendfile", True) + duplicateSendfile = self.local.db.get("duplicateSendfile", True) if not os.path.isfile(filePath): raise Exception("SendFile error: no such file '{filePath}'".format(filePath=filePath)) if timeout and wallet: @@ -1439,7 +1206,7 @@ def SendFile(self, filePath, wallet=None, **kwargs): #end define def WaitTransaction(self, wallet, timeout=30): - local.add_log("start WaitTransaction function", "debug") + self.local.add_log("start WaitTransaction function", "debug") timesleep = 3 steps = timeout // timesleep for i in range(steps): @@ -1451,7 +1218,7 @@ def WaitTransaction(self, wallet, timeout=30): #end define def GetReturnedStake(self, fullElectorAddr, inputAddr): - local.add_log("start GetReturnedStake function", "debug") + self.local.add_log("start GetReturnedStake function", "debug") workchain, addr = self.ParseInputAddr(inputAddr) cmd = f"runmethodfull {fullElectorAddr} compute_returned_stake 0x{addr}" result = self.liteClient.Run(cmd) @@ -1463,7 +1230,7 @@ def GetReturnedStake(self, fullElectorAddr, inputAddr): #end define def ProcessRecoverStake(self): - local.add_log("start ProcessRecoverStake function", "debug") + self.local.add_log("start ProcessRecoverStake function", "debug") resultFilePath = self.tempDir + self.nodeName + "recover-query" args = ["recover-stake.fif", resultFilePath] result = self.fift.Run(args) @@ -1472,9 +1239,9 @@ def ProcessRecoverStake(self): #end define def GetStake(self, account, args=None): - stake = local.db.get("stake") - usePool = local.db.get("usePool") - stakePercent = local.db.get("stakePercent", 99) + stake = self.local.db.get("stake") + usePool = self.local.db.get("usePool") + stakePercent = self.local.db.get("stakePercent", 99) vconfig = self.GetValidatorConfig() config17 = self.GetConfig17() @@ -1489,7 +1256,7 @@ def GetStake(self, account, args=None): # Stake was a number stake = int(desiredStake) else: - local.add_log("Specified stake must be a percentage or whole number", "error") + self.local.add_log("Specified stake must be a percentage or whole number", "error") return # Limit stake to maximum available amount minus 10 (for transaction fees) @@ -1502,7 +1269,7 @@ def GetStake(self, account, args=None): if stake is None: sp = stakePercent / 100 if sp > 1 or sp < 0: - local.add_log("Wrong stakePercent value. Using default stake.", "warning") + self.local.add_log("Wrong stakePercent value. Using default stake.", "warning") elif len(vconfig.validators) == 0: stake = int(account.balance*sp/2) elif len(vconfig.validators) > 0: @@ -1512,15 +1279,15 @@ def GetStake(self, account, args=None): # Check if we have enough coins if stake > config17["maxStake"]: text = "Stake is greater than the maximum value. Will be used the maximum stake." - local.add_log(text, "warning") + self.local.add_log(text, "warning") stake = config17["maxStake"] if config17["minStake"] > stake: text = "Stake less than the minimum stake. Minimum stake: {minStake}".format(minStake=config17["minStake"]) - #local.add_log(text, "error") + # self.local.add_log(text, "error") raise Exception(text) if stake > account.balance: text = "Don't have enough coins. stake: {stake}, account balance: {balance}".format(stake=stake, balance=account.balance) - #local.add_log(text, "error") + # self.local.add_log(text, "error") raise Exception(text) #end if @@ -1529,7 +1296,7 @@ def GetStake(self, account, args=None): def GetMaxFactor(self): # Either use defined maxFactor, or set maximal allowed by config17 - maxFactor = local.db.get("maxFactor") + maxFactor = self.local.db.get("maxFactor") if maxFactor is None: config17 = self.GetConfig17() maxFactor = config17["maxStakeFactor"] / 65536 @@ -1537,15 +1304,53 @@ def GetMaxFactor(self): return maxFactor #end define + def GetNominationControllerLastSentStakeTime(self, addrB64): + cmd = f"runmethodfull {addrB64} all_data" + result = self.liteClient.Run(cmd) + buff = self.Result2List(result) + return buff[-1] + #end define + + def IsNominationControllerReadyToStake(self, addrB64): + now = GetTimestamp() + config15 = self.GetConfig15() + lastSentStakeTime = self.GetNominationControllerLastSentStakeTime(addrB64) + stakeFreezeDelay = config15["validatorsElectedFor"] + config15["stakeHeldFor"] + result = lastSentStakeTime + stakeFreezeDelay < now + return result + #end define + + def IsNominationControllerReadyToVote(self, addrB64): + vwl = self.GetValidatorsWalletsList() + result = addrB64 in vwl + return result + #end define + + def GetNominationController(self, mode): + self.local.AddLog("start GetNominationController function", "debug") + nominationControllerList = ["nomination_controller_001", "nomination_controller_002"] + for item in nominationControllerList: + wallet = self.GetLocalWallet(item) + if mode == "stake" and self.IsNominationControllerReadyToStake(wallet.addrB64): + return wallet + if mode == "vote" and self.IsNominationControllerReadyToVote(wallet.addrB64): + return wallet + raise Exception("Validator сontroller not found") + #end define + def GetValidatorWallet(self, mode="stake"): - local.add_log("start GetValidatorWallet function", "debug") - walletName = local.db.get("validatorWalletName") - wallet = self.GetLocalWallet(walletName) + self.local.add_log("start GetValidatorWallet function", "debug") + useNominationController = self.local.db.get("useNominationController") + if useNominationController is True: + wallet = self.GetNominationController(mode) + else: + walletName = self.local.db.get("validatorWalletName") + wallet = self.GetLocalWallet(walletName) return wallet #end define def ElectionEntry(self, args=None): - usePool = local.db.get("usePool") + usePool = self.local.db.get("usePool") wallet = self.GetValidatorWallet() addrB64 = wallet.addrB64 if wallet is None: @@ -1557,12 +1362,12 @@ def ElectionEntry(self, args=None): addrB64 = pool.addrB64 #end if - local.add_log("start ElectionEntry function", "debug") + self.local.add_log("start ElectionEntry function", "debug") # Check if validator is not synchronized validatorStatus = self.GetValidatorStatus() validatorOutOfSync = validatorStatus.get("outOfSync") if validatorOutOfSync > 60: - local.add_log("Validator is not synchronized", "error") + self.local.add_log("Validator is not synchronized", "error") return #end if @@ -1572,7 +1377,7 @@ def ElectionEntry(self, args=None): # Check if elections started if (startWorkTime == 0): - local.add_log("Elections have not yet begun", "info") + self.local.add_log("Elections have not yet begun", "info") return #end if @@ -1580,15 +1385,15 @@ def ElectionEntry(self, args=None): adnlAddr = self.GetAdnlAddr() # Check wether it is too early to participate - if "participateBeforeEnd" in local.db: + if "participateBeforeEnd" in self.local.db: now = time.time() - if (startWorkTime - now) > local.db["participateBeforeEnd"] and \ - (now + local.db["periods"]["elections"]) < startWorkTime: + if (startWorkTime - now) > self.local.db["participateBeforeEnd"] and \ + (now + self.local.db["periods"]["elections"]) < startWorkTime: return # Check if election entry already completed entries = self.GetElectionEntries() if adnlAddr in entries: - local.add_log("Elections entry already completed", "info") + self.local.add_log("Elections entry already completed", "info") return #end if @@ -1632,11 +1437,11 @@ def ElectionEntry(self, args=None): # Save vars to json file self.SaveElectionVarsToJsonFile(wallet=wallet, account=account, stake=stake, maxFactor=maxFactor, fullElectorAddr=fullElectorAddr, startWorkTime=startWorkTime, validatorsElectedFor=validatorsElectedFor, endWorkTime=endWorkTime, validatorKey=validatorKey, validatorPubkey_b64=validatorPubkey_b64, adnlAddr=adnlAddr, var1=var1, validatorSignature=validatorSignature, validatorPubkey=validatorPubkey) - local.add_log("ElectionEntry completed. Start work time: " + str(startWorkTime)) + self.local.add_log("ElectionEntry completed. Start work time: " + str(startWorkTime)) #end define def GetValidatorKeyByTime(self, startWorkTime, endWorkTime): - local.add_log("start GetValidatorKeyByTime function", "debug") + self.local.add_log("start GetValidatorKeyByTime function", "debug") # Check temp key vconfig = self.GetValidatorConfig() for item in vconfig.validators: @@ -1660,18 +1465,18 @@ def RecoverStake(self): raise Exception("Validator wallet not found") #end if - local.add_log("start RecoverStake function", "debug") + self.local.add_log("start RecoverStake function", "debug") fullElectorAddr = self.GetFullElectorAddr() returnedStake = self.GetReturnedStake(fullElectorAddr, wallet.addrB64) if returnedStake == 0: - local.add_log("You have nothing on the return stake", "debug") + self.local.add_log("You have nothing on the return stake", "debug") return #end if resultFilePath = self.ProcessRecoverStake() resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, fullElectorAddr, 1) self.SendFile(resultFilePath, wallet) - local.add_log("RecoverStake completed") + self.local.add_log("RecoverStake completed") #end define def PoolRecoverStake(self, poolAddr): @@ -1680,15 +1485,15 @@ def PoolRecoverStake(self, poolAddr): raise Exception("Validator wallet not found") #end if - local.add_log("start PoolRecoverStake function", "debug") + self.local.add_log("start PoolRecoverStake function", "debug") resultFilePath = self.PoolProcessRecoverStake() resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, poolAddr, 1.2) self.SendFile(resultFilePath, wallet) - local.add_log("PoolRecoverStake completed") + self.local.add_log("PoolRecoverStake completed") #end define def PoolsUpdateValidatorSet(self): - local.add_log("start PoolsUpdateValidatorSet function", "debug") + self.local.add_log("start PoolsUpdateValidatorSet function", "debug") wallet = self.GetValidatorWallet() pools = self.GetPools() for pool in pools: @@ -1696,7 +1501,7 @@ def PoolsUpdateValidatorSet(self): #end define def PoolUpdateValidatorSet(self, pool, wallet): - local.add_log("start PoolUpdateValidatorSet function", "debug") + self.local.add_log("start PoolUpdateValidatorSet function", "debug") poolAddr = pool.addrB64 poolData = self.GetPoolData(poolAddr) if poolData is None: @@ -1727,7 +1532,7 @@ def PoolUpdateValidatorSet(self, pool, wallet): #end define def PoolProcessUpdateValidatorSet(self, poolAddr, wallet): - local.add_log("start PoolProcessUpdateValidatorSet function", "debug") + self.local.add_log("start PoolProcessUpdateValidatorSet function", "debug") resultFilePath = self.tempDir + "pool-update-validator-set-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/update-validator-set.fif" args = [fiftScript, resultFilePath] @@ -1735,19 +1540,19 @@ def PoolProcessUpdateValidatorSet(self, poolAddr, wallet): resultFilePath = parse(result, "Saved to file ", '\n') resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, poolAddr, 1.1) self.SendFile(resultFilePath, wallet) - local.add_log("PoolProcessUpdateValidatorSet completed") + self.local.add_log("PoolProcessUpdateValidatorSet completed") #end define def PoolWithdrawRequests(self, pool, wallet): - local.add_log("start PoolWithdrawRequests function", "debug") + self.local.add_log("start PoolWithdrawRequests function", "debug") resultFilePath = self.PoolProcessWihtdrawRequests() resultFilePath = self.SignBocWithWallet(wallet, resultFilePath, pool.addrB64, 10) self.SendFile(resultFilePath, wallet) - local.add_log("PoolWithdrawRequests completed") + self.local.add_log("PoolWithdrawRequests completed") #end define def PoolProcessWihtdrawRequests(self): - local.add_log("start PoolProcessWihtdrawRequests function", "debug") + self.local.add_log("start PoolProcessWihtdrawRequests function", "debug") resultFilePath = self.tempDir + "pool-withdraw-requests-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/process-withdraw-requests.fif" args = [fiftScript, resultFilePath] @@ -1768,7 +1573,7 @@ def HasPoolWithdrawRequests(self, pool): #end define def SaveElectionVarsToJsonFile(self, **kwargs): - local.add_log("start SaveElectionVarsToJsonFile function", "debug") + self.local.add_log("start SaveElectionVarsToJsonFile function", "debug") fileName = self.tempDir + self.nodeName + str(kwargs.get("startWorkTime")) + "_ElectionEntry.json" wallet = kwargs.get("wallet") account = kwargs.get("account") @@ -1783,12 +1588,12 @@ def SaveElectionVarsToJsonFile(self, **kwargs): #ned define def CreateWallet(self, name, workchain=0, version="v1", **kwargs): - local.add_log("start CreateWallet function", "debug") + self.local.add_log("start CreateWallet function", "debug") subwalletDefault = 698983191 + workchain # 0x29A9A317 + workchain subwallet = kwargs.get("subwallet", subwalletDefault) walletPath = self.walletsDir + name if os.path.isfile(walletPath + ".pk") and "v3" not in version: - local.add_log("CreateWallet error: Wallet already exists: " + name, "warning") + self.local.add_log("CreateWallet error: Wallet already exists: " + name, "warning") else: if "v1" in version: fiftScript = "new-wallet.fif" @@ -1813,10 +1618,10 @@ def CreateHighWallet(self, name, **kwargs): subwalletDefault = 698983191 + workchain # 0x29A9A317 + workchain subwallet = kwargs.get("subwallet", subwalletDefault) version = kwargs.get("version", "hv1") - local.add_log("start CreateHighWallet function", "debug") + self.local.AddLog("start CreateHighWallet function", "debug") walletPath = self.walletsDir + name if os.path.isfile(walletPath + ".pk") and os.path.isfile(walletPath + str(subwallet) + ".addr"): - local.add_log("CreateHighWallet error: Wallet already exists: " + name + str(subwallet), "warning") + self.local.AddLog("CreateHighWallet error: Wallet already exists: " + name + str(subwallet), "warning") else: args = ["new-highload-wallet.fif", workchain, subwallet, walletPath] result = self.fift.Run(args) @@ -1829,12 +1634,12 @@ def CreateHighWallet(self, name, **kwargs): #end define def ActivateWallet(self, wallet): - local.add_log("start ActivateWallet function", "debug") + self.local.add_log("start ActivateWallet function", "debug") account = self.GetAccount(wallet.addrB64) if account.status == "empty": raise Exception("ActivateWallet error: account status is empty") elif account.status == "active": - local.add_log("ActivateWallet warning: account status is active", "warning") + self.local.add_log("ActivateWallet warning: account status is active", "warning") else: self.SendFile(wallet.bocFilePath, wallet, remove=False) #end define @@ -1870,7 +1675,7 @@ def ExportWallet(self, walletName): #end define def GetWalletsNameList(self): - local.add_log("start GetWalletsNameList function", "debug") + self.local.add_log("start GetWalletsNameList function", "debug") walletsNameList = list() for fileName in os.listdir(self.walletsDir): if fileName.endswith(".addr"): @@ -1883,7 +1688,7 @@ def GetWalletsNameList(self): #end define def GetWallets(self): - local.add_log("start GetWallets function", "debug") + self.local.add_log("start GetWallets function", "debug") wallets = list() walletsNameList = self.GetWalletsNameList() for walletName in walletsNameList: @@ -1893,7 +1698,7 @@ def GetWallets(self): #end define def GenerateWalletName(self): - local.add_log("start GenerateWalletName function", "debug") + self.local.add_log("start GenerateWalletName function", "debug") index = 1 index_str = str(index).rjust(3, '0') walletPrefix = "wallet_" @@ -1915,7 +1720,7 @@ def GenerateWalletName(self): #end define def WalletsCheck(self): - local.add_log("start WalletsCheck function", "debug") + self.local.add_log("start WalletsCheck function", "debug") wallets = self.GetWallets() for wallet in wallets: if os.path.isfile(wallet.bocFilePath): @@ -1925,7 +1730,7 @@ def WalletsCheck(self): #end define def GetValidatorConfig(self): - #local.add_log("start GetValidatorConfig function", "debug") + #self.local.add_log("start GetValidatorConfig function", "debug") result = self.validatorConsole.Run("getconfig") text = parse(result, "---------", "--------") vconfig = json.loads(text) @@ -1933,8 +1738,8 @@ def GetValidatorConfig(self): #end define def GetOverlaysStats(self): - local.add_log("start GetOverlaysStats function", "debug") - resultFilePath = local.buffer.my_temp_dir + "getoverlaysstats.json" + self.local.add_log("start GetOverlaysStats function", "debug") + resultFilePath = self.local.buffer.my_temp_dir + "getoverlaysstats.json" result = self.validatorConsole.Run(f"getoverlaysstatsjson {resultFilePath}") if "wrote stats" not in result: raise Exception(f"GetOverlaysStats error: {result}") @@ -1958,7 +1763,7 @@ def GetWalletId(self, wallet): #end define def MoveCoins(self, wallet, dest, coins, **kwargs): - local.add_log("start MoveCoins function", "debug") + self.local.add_log("start MoveCoins function", "debug") flags = kwargs.get("flags", list()) timeout = kwargs.get("timeout", 30) subwallet = kwargs.get("subwallet") @@ -1989,13 +1794,13 @@ def MoveCoins(self, wallet, dest, coins, **kwargs): if bounceable == False and destAccount.status == "active": flags += ["-b"] text = "Find non-bounceable flag, but destination account already active. Using bounceable flag" - local.add_log(text, "warning") + self.local.add_log(text, "warning") elif "-n" not in flags and bounceable == True and destAccount.status != "active": raise Exception("Find bounceable flag, but destination account is not active. Use non-bounceable address or flag -n") #end if seqno = self.GetSeqno(wallet) - resultFilePath = local.buffer.my_temp_dir + wallet.name + "_wallet-query" + resultFilePath = self.local.buffer.my_temp_dir + wallet.name + "_wallet-query" if "v1" in wallet.version: fiftScript = "wallet.fif" args = [fiftScript, wallet.path, dest, seqno, coins, "-m", mode, resultFilePath] @@ -2013,7 +1818,7 @@ def MoveCoins(self, wallet, dest, coins, **kwargs): #end define def MoveCoinsThroughProxy(self, wallet, dest, coins): - local.add_log("start MoveCoinsThroughProxy function", "debug") + self.local.add_log("start MoveCoinsThroughProxy function", "debug") wallet1 = self.CreateWallet("proxy_wallet1", 0) wallet2 = self.CreateWallet("proxy_wallet2", 0) self.MoveCoins(wallet, wallet1.addrB64_init, coins) @@ -2026,16 +1831,16 @@ def MoveCoinsThroughProxy(self, wallet, dest, coins): #end define def MoveCoinsFromHW(self, wallet, destList, **kwargs): - local.add_log("start MoveCoinsFromHW function", "debug") + self.local.add_log("start MoveCoinsFromHW function", "debug") flags = kwargs.get("flags") timeout = kwargs.get("timeout", 30) if len(destList) == 0: - local.add_log("MoveCoinsFromHW warning: destList is empty, break function", "warning") + self.local.add_log("MoveCoinsFromHW warning: destList is empty, break function", "warning") return #end if - orderFilePath = local.buffer.my_temp_dir + wallet.name + "_order.txt" + orderFilePath = self.local.buffer.my_temp_dir + wallet.name + "_order.txt" lines = list() for dest, coins in destList: lines.append("SEND {dest} {coins}".format(dest=dest, coins=coins)) @@ -2049,7 +1854,7 @@ def MoveCoinsFromHW(self, wallet, destList, **kwargs): elif "v2" in wallet.version: fiftScript = "highload-wallet-v2.fif" seqno = self.GetSeqno(wallet) - resultFilePath = local.buffer.my_temp_dir + wallet.name + "_wallet-query" + resultFilePath = self.local.buffer.my_temp_dir + wallet.name + "_wallet-query" args = [fiftScript, wallet.path, wallet.subwallet, seqno, orderFilePath, resultFilePath] if flags: args += flags @@ -2060,7 +1865,7 @@ def MoveCoinsFromHW(self, wallet, destList, **kwargs): def GetValidatorKey(self): vconfig = self.GetValidatorConfig() - for validator in vconfig.validators: + for validator in vconfig["validators"]: validatorId = validator["id"] key_bytes = base64.b64decode(validatorId) validatorKey = key_bytes.hex().upper() @@ -2098,7 +1903,7 @@ def GetElectionEntries(self, past=False): #end if # Get raw data - local.add_log("start GetElectionEntries function", "debug") + self.local.add_log("start GetElectionEntries function", "debug") cmd = "runmethodfull {fullElectorAddr} participant_list_extended".format(fullElectorAddr=fullElectorAddr) result = self.liteClient.Run(cmd) rawElectionEntries = self.Result2List(result) @@ -2140,10 +1945,10 @@ def GetElectionEntries(self, past=False): def GetSaveElections(self): timestamp = get_timestamp() - saveElections = local.db.get("saveElections") + saveElections = self.local.db.get("saveElections") if saveElections is None: saveElections = dict() - local.db["saveElections"] = saveElections + self.local.db["saveElections"] = saveElections buff = saveElections.copy() for key, item in buff.items(): diffTime = timestamp - int(key) @@ -2160,7 +1965,7 @@ def GetSaveElectionEntries(self, electionId): #end define def GetOffers(self): - local.add_log("start GetOffers function", "debug") + self.local.add_log("start GetOffers function", "debug") fullConfigAddr = self.GetFullConfigAddr() # Get raw data cmd = "runmethodfull {fullConfigAddr} list_proposals".format(fullConfigAddr=fullConfigAddr) @@ -2212,7 +2017,7 @@ def GetOffers(self): #end define def GetOfferDiff(self, offerHash): - local.add_log("start GetOfferDiff function", "debug") + self.local.add_log("start GetOfferDiff function", "debug") offer = self.GetOffer(offerHash) configId = offer["config"]["id"] configValue = offer["config"]["value"] @@ -2309,7 +2114,7 @@ def GetComplaints(self, electionId=None, past=False): #end if # Get raw data - local.add_log("start GetComplaints function", "debug") + self.local.add_log("start GetComplaints function", "debug") cmd = "runmethodfull {fullElectorAddr} list_complaints {electionId}".format(fullElectorAddr=fullElectorAddr, electionId=electionId) result = self.liteClient.Run(cmd) rawComplaints = self.Result2List(result) @@ -2377,10 +2182,10 @@ def GetComplaints(self, electionId=None, past=False): def GetSaveComplaints(self): timestamp = get_timestamp() - saveComplaints = local.db.get("saveComplaints") + saveComplaints = self.local.db.get("saveComplaints") if type(saveComplaints) is not dict: saveComplaints = dict() - local.db["saveComplaints"] = saveComplaints + self.local.db["saveComplaints"] = saveComplaints buff = saveComplaints.copy() for key, item in buff.items(): diffTime = timestamp - int(key) @@ -2400,7 +2205,7 @@ def GetAdnlFromPubkey(self, inputPubkey): #end define def GetComplaintsNumber(self): - local.add_log("start GetComplaintsNumber function", "debug") + self.local.add_log("start GetComplaintsNumber function", "debug") result = dict() complaints = self.GetComplaints() votedComplaints = self.GetVotedComplaints() @@ -2418,7 +2223,7 @@ def GetComplaintsNumber(self): #end define def GetComplaint(self, electionId, complaintHash): - local.add_log("start GetComplaint function", "debug") + self.local.add_log("start GetComplaint function", "debug") complaints = self.GetComplaints(electionId) for key, item in complaints.items(): if complaintHash == item.get("hash"): @@ -2427,7 +2232,7 @@ def GetComplaint(self, electionId, complaintHash): #end define def SignProposalVoteRequestWithValidator(self, offerHash, validatorIndex, validatorPubkey_b64, validatorSignature): - local.add_log("start SignProposalVoteRequestWithValidator function", "debug") + self.local.add_log("start SignProposalVoteRequestWithValidator function", "debug") fileName = self.tempDir + self.nodeName + "proposal_vote-msg-body.boc" args = ["config-proposal-vote-signed.fif", "-i", validatorIndex, offerHash, validatorPubkey_b64, validatorSignature, fileName] result = self.fift.Run(args) @@ -2436,7 +2241,7 @@ def SignProposalVoteRequestWithValidator(self, offerHash, validatorIndex, valida #end define def SignComplaintVoteRequestWithValidator(self, complaintHash, electionId, validatorIndex, validatorPubkey_b64, validatorSignature): - local.add_log("start SignComplaintRequestWithValidator function", "debug") + self.local.add_log("start SignComplaintRequestWithValidator function", "debug") fileName = self.tempDir + "complaint_vote-msg-body.boc" args = ["complaint-vote-signed.fif", validatorIndex, electionId, complaintHash, validatorPubkey_b64, validatorSignature, fileName] result = self.fift.Run(args) @@ -2445,7 +2250,7 @@ def SignComplaintVoteRequestWithValidator(self, complaintHash, electionId, valid #end define def VoteOffer(self, offerHash): - local.add_log("start VoteOffer function", "debug") + self.local.add_log("start VoteOffer function", "debug") fullConfigAddr = self.GetFullConfigAddr() wallet = self.GetValidatorWallet(mode="vote") validatorKey = self.GetValidatorKey() @@ -2453,7 +2258,7 @@ def VoteOffer(self, offerHash): validatorIndex = self.GetValidatorIndex() offer = self.GetOffer(offerHash) if validatorIndex in offer.get("votedValidators"): - local.add_log("Proposal already has been voted", "debug") + self.local.add_log("Proposal already has been voted", "debug") return var1 = self.CreateConfigProposalRequest(offerHash, validatorIndex) validatorSignature = self.GetValidatorSignature(validatorKey, var1) @@ -2464,7 +2269,7 @@ def VoteOffer(self, offerHash): #end define def VoteComplaint(self, electionId, complaintHash): - local.add_log("start VoteComplaint function", "debug") + self.local.add_log("start VoteComplaint function", "debug") complaintHash = int(complaintHash) fullElectorAddr = self.GetFullElectorAddr() wallet = self.GetValidatorWallet(mode="vote") @@ -2475,7 +2280,7 @@ def VoteComplaint(self, electionId, complaintHash): votedValidators = complaint.get("votedValidators") pubkey = complaint.get("pubkey") if validatorIndex in votedValidators: - local.add_log("Complaint already has been voted", "info") + self.local.add_log("Complaint already has been voted", "info") return var1 = self.CreateComplaintRequest(electionId, complaintHash, validatorIndex) validatorSignature = self.GetValidatorSignature(validatorKey, var1) @@ -2486,7 +2291,7 @@ def VoteComplaint(self, electionId, complaintHash): #end define def SaveComplaints(self, electionId): - local.add_log("start SaveComplaints function", "debug") + self.local.add_log("start SaveComplaints function", "debug") filePrefix = self.tempDir + "scheck_" cmd = "savecomplaints {electionId} {filePrefix}".format(electionId=electionId, filePrefix=filePrefix) result = self.liteClient.Run(cmd) @@ -2506,7 +2311,7 @@ def SaveComplaints(self, electionId): #end define def CheckComplaint(self, filePath): - local.add_log("start CheckComplaint function", "debug") + self.local.add_log("start CheckComplaint function", "debug") cmd = "loadproofcheck {filePath}".format(filePath=filePath) result = self.liteClient.Run(cmd, timeout=30) lines = result.split('\n') @@ -2542,7 +2347,7 @@ def GetValidatorsLoad(self, start, end, saveCompFiles=False): #end if text = "start GetValidatorsLoad function ({}, {})".format(start, end) - local.add_log(text, "debug") + self.local.add_log(text, "debug") if saveCompFiles is True: filePrefix = self.tempDir + f"checkload_{start}_{end}" else: @@ -2655,7 +2460,7 @@ def GetValidatorsList(self, past=False): #end define def CheckValidators(self, start, end): - local.add_log("start CheckValidators function", "debug") + self.local.add_log("start CheckValidators function", "debug") electionId = start complaints = self.GetComplaints(electionId) data = self.GetValidatorsLoad(start, end, saveCompFiles=True) @@ -2682,11 +2487,11 @@ def CheckValidators(self, start, end): fileName = self.PrepareComplaint(electionId, fileName) fileName = self.SignBocWithWallet(wallet, fileName, fullElectorAddr, 300) self.SendFile(fileName, wallet) - local.add_log("var1: {}, var2: {}, pubkey: {}, election_id: {}".format(var1, var2, pubkey, electionId), "debug") + self.local.add_log("var1: {}, var2: {}, pubkey: {}, election_id: {}".format(var1, var2, pubkey, electionId), "debug") #end define def GetOffer(self, offerHash): - local.add_log("start GetOffer function", "debug") + self.local.add_log("start GetOffer function", "debug") offers = self.GetOffers() for offer in offers: if offerHash == offer.get("hash"): @@ -2695,7 +2500,7 @@ def GetOffer(self, offerHash): #end define def GetOffersNumber(self): - local.add_log("start GetOffersNumber function", "debug") + self.local.add_log("start GetOffersNumber function", "debug") result = dict() offers = self.GetOffers() saveOffers = self.GetSaveOffers() @@ -2721,12 +2526,12 @@ def GetValidatorIndex(self, adnlAddr=None): if adnlAddr == searchAdnlAddr: return index index += 1 - local.add_log("GetValidatorIndex warning: index not found.", "warning") + self.local.add_log("GetValidatorIndex warning: index not found.", "warning") return -1 #end define def GetValidatorEfficiency(self, adnlAddr=None): - local.add_log("start GetValidatorEfficiency function", "debug") + self.local.add_log("start GetValidatorEfficiency function", "debug") validators = self.GetValidatorsList() if adnlAddr is None: adnlAddr = self.GetAdnlAddr() @@ -2735,7 +2540,7 @@ def GetValidatorEfficiency(self, adnlAddr=None): if adnlAddr == searchAdnlAddr: efficiency = validator.get("efficiency") return efficiency - local.add_log("GetValidatorEfficiency warning: efficiency not found.", "warning") + self.local.add_log("GetValidatorEfficiency warning: efficiency not found.", "warning") #end define def GetDbUsage(self): @@ -2745,7 +2550,7 @@ def GetDbUsage(self): #end define def GetDbSize(self, exceptions="log"): - local.add_log("start GetDbSize function", "debug") + self.local.add_log("start GetDbSize function", "debug") exceptions = exceptions.split() totalSize = 0 path = "/var/ton-work/" @@ -2906,8 +2711,18 @@ def GetDomainFromAuction(self, walletName, addr): self.SendFile(resultFilePath, wallet) #end define + def GetDomainFromAuction(self, walletName, addr): + wallet = self.GetLocalWallet(walletName) + bocPath = self.local.buffer.my_temp_dir + "get_dns_data.boc" + bocData = bytes.fromhex("b5ee9c7241010101000e0000182fcb26a20000000000000000f36cae4d") + with open(bocPath, 'wb') as file: + file.write(bocData) + resultFilePath = self.SignBocWithWallet(wallet, bocPath, addr, 0.1) + self.SendFile(resultFilePath, wallet) + #end define + def NewDomain(self, domain): - local.add_log("start NewDomain function", "debug") + self.local.add_log("start NewDomain function", "debug") domainName = domain["name"] buff = domainName.split('.') subdomain = buff.pop(0) @@ -2933,15 +2748,15 @@ def NewDomain(self, domain): #end define def AddDomain(self, domain): - if "domains" not in local.db: - local.db["domains"] = list() + if "domains" not in self.local.db: + self.local.db["domains"] = list() #end if - local.db["domains"].append(domain) - local.save() + self.local.db["domains"].append(domain) + self.local.save() #end define def GetDomains(self): - domains = local.db.get("domains", list()) + domains = self.local.db.get("domains", list()) for domain in domains: domainName = domain.get("name") domain["endTime"] = self.GetDomainEndTime(domainName) @@ -2957,39 +2772,39 @@ def GetDomain(self, domainName): #end define def DeleteDomain(self, domainName): - domains = local.db.get("domains") + domains = self.local.db.get("domains") for domain in domains: if (domainName == domain.get("name")): domains.remove(domain) - local.save() + self.local.save() return raise Exception("DeleteDomain error: Domain not found") #end define def GetAutoTransferRules(self): - autoTransferRules = local.db.get("autoTransferRules") + autoTransferRules = self.local.db.get("autoTransferRules") if autoTransferRules is None: autoTransferRules = list() - local.db["autoTransferRules"] = autoTransferRules + self.local.db["autoTransferRules"] = autoTransferRules return autoTransferRules #end define def AddAutoTransferRule(self, rule): autoTransferRules = self.GetAutoTransferRules() autoTransferRules.append(rule) - local.save() + self.local.save() #end define def AddBookmark(self, bookmark): - if "bookmarks" not in local.db: - local.db["bookmarks"] = list() + if "bookmarks" not in self.local.db: + self.local.db["bookmarks"] = list() #end if - local.db["bookmarks"].append(bookmark) - local.save() + self.local.db["bookmarks"].append(bookmark) + self.local.save() #end define def GetBookmarks(self): - bookmarks = local.db.get("bookmarks") + bookmarks = self.local.db.get("bookmarks") if bookmarks is not None: for bookmark in bookmarks: self.WriteBookmarkData(bookmark) @@ -2997,7 +2812,7 @@ def GetBookmarks(self): #end define def GetBookmarkAddr(self, type, name): - bookmarks = local.db.get("bookmarks", list()) + bookmarks = self.local.db.get("bookmarks", list()) for bookmark in bookmarks: bookmarkType = bookmark.get("type") bookmarkName = bookmark.get("name") @@ -3008,13 +2823,13 @@ def GetBookmarkAddr(self, type, name): #end define def DeleteBookmark(self, name, type): - bookmarks = local.db.get("bookmarks") + bookmarks = self.local.db.get("bookmarks") for bookmark in bookmarks: bookmarkType = bookmark.get("type") bookmarkName = bookmark.get("name") if (type == bookmarkType and name == bookmarkName): bookmarks.remove(bookmark) - local.save() + self.local.save() return raise Exception("DeleteBookmark error: Bookmark not found") #end define @@ -3042,10 +2857,10 @@ def WriteBookmarkData(self, bookmark): def GetSaveOffers(self): bname = "saveOffers" - saveOffers = local.db.get(bname) - if type(saveOffers) != dict: - saveOffers = dict() - local.db[bname] = saveOffers + saveOffers = self.local.db.get(bname) + if saveOffers is None: + saveOffers = list() + self.local.db[bname] = saveOffers return saveOffers #end define @@ -3054,16 +2869,16 @@ def AddSaveOffer(self, offer): offerPseudohash = offer.get("pseudohash") saveOffers = self.GetSaveOffers() if offerHash not in saveOffers: - saveOffers[offerHash] = offerPseudohash - local.save() + saveOffers.append(offerHash) + self.local.save() #end define def GetVotedComplaints(self): bname = "votedComplaints" - votedComplaints = local.db.get(bname) + votedComplaints = self.local.db.get(bname) if votedComplaints is None: votedComplaints = dict() - local.db[bname] = votedComplaints + self.local.db[bname] = votedComplaints return votedComplaints #end define @@ -3072,7 +2887,7 @@ def AddVotedComplaints(self, complaint): votedComplaints = self.GetVotedComplaints() if pseudohash not in votedComplaints: votedComplaints[pseudohash] = complaint - local.save() + self.local.save() #end define def GetDestinationAddr(self, destination): @@ -3138,7 +2953,7 @@ def ParseAddrB64(self, addrB64): networkTestnet = self.IsTestnet() if testnet != networkTestnet: text = f"ParseAddrB64 warning: testnet flag do not match. Addr: {testnet}, Network: {networkTestnet}" - local.add_log(text, "warning") + self.local.add_log(text, "warning") #end if # get wc and addr @@ -3147,7 +2962,8 @@ def ParseAddrB64(self, addrB64): crc_bytes = b[34:36] crc_data = bytes(b[:34]) crc = int.from_bytes(crc_bytes, "big") - check_crc = crc16.xmodem(crc_data) + # check_crc = crc16.crc16xmodem(crc_data) + check_crc = fastcrc.crc16.xmodem(crc_data) # TODO: check this library! if crc != check_crc: raise Exception("ParseAddrB64 error: crc do not match") #end if @@ -3192,7 +3008,7 @@ def IsBounceableAddrB64(self, inputAddr): def GetNetLoadAvg(self, statistics=None): if statistics is None: - statistics = local.db.get("statistics") + statistics = self.local.db.get("statistics") if statistics: netLoadAvg = statistics.get("netLoadAvg") else: @@ -3202,7 +3018,7 @@ def GetNetLoadAvg(self, statistics=None): def GetTpsAvg(self, statistics=None): if statistics is None: - statistics = local.db.get("statistics") + statistics = self.local.db.get("statistics") if statistics: tpsAvg = statistics.get("tpsAvg") else: @@ -3212,7 +3028,7 @@ def GetTpsAvg(self, statistics=None): def GetStatistics(self, name, statistics=None): if statistics is None: - statistics = local.db.get("statistics") + statistics = self.local.db.get("statistics") if statistics: data = statistics.get(name) else: @@ -3221,8 +3037,8 @@ def GetStatistics(self, name, statistics=None): #end define def GetSettings(self, name): - #local.load_db() - result = local.db.get(name) + # self.local.load_db() + result = self.local.db.get(name) return result #end define @@ -3230,8 +3046,8 @@ def SetSettings(self, name, data): try: data = json.loads(data) except: pass - local.db[name] = data - local.save() + self.local.db[name] = data + self.local.save() #end define def Tlb2Json(self, text): @@ -3305,7 +3121,7 @@ def Tlb2Json(self, text): #end define def SignShardOverlayCert(self, adnl, pubkey): - local.add_log("start SignShardOverlayCert function", "debug") + self.local.add_log("start SignShardOverlayCert function", "debug") fileName = self.tempDir + pubkey + ".cert" cmd = "signshardoverlaycert {workchain} {shardprefix} {pubkey} {expireat} {maxsize} {outfile}" cmd = cmd.format(workchain=-1, shardprefix=-9223372036854775808, pubkey=pubkey, expireat=172800, maxsize=8192, outfile=fileName) @@ -3327,7 +3143,7 @@ def SignShardOverlayCert(self, adnl, pubkey): #end define def ImportShardOverlayCert(self): - local.add_log("start ImportShardOverlayCert function", "debug") + self.local.add_log("start ImportShardOverlayCert function", "debug") adnlAddr = self.GetAdnlAddr() pubkey = self.GetPubKey(adnlAddr) adnl = pubkey # adnl = adnlAddr @@ -3352,7 +3168,7 @@ def ImportShardOverlayCert(self): # Check certificate if cert is None: - local.add_log("ImportShardOverlayCert warning: certificate not found", "warning") + self.local.add_log("ImportShardOverlayCert warning: certificate not found", "warning") return #end if @@ -3364,7 +3180,7 @@ def ImportShardOverlayCert(self): #end define def ImportCertificate(self, pubkey, fileName): - local.add_log("start ImportCertificate function", "debug") + self.local.add_log("start ImportCertificate function", "debug") cmd = "importshardoverlaycert {workchain} {shardprefix} {pubkey} {certfile}" cmd = cmd.format(workchain=-1, shardprefix=-9223372036854775808, pubkey=pubkey, certfile=fileName) result = self.validatorConsole.Run(cmd) @@ -3380,7 +3196,7 @@ def GetValidatorsWalletsList(self): #end define def DownloadContract(self, url, branch=None): - local.add_log("start DownloadContract function", "debug") + self.local.add_log("start DownloadContract function", "debug") buff = url.split('/') gitPath = self.contractsDir + buff[-1] + '/' @@ -3411,8 +3227,73 @@ def DownloadContract(self, url, branch=None): #end if #end define + def CreateNominationController(self, name, nominatorAddr, **kwargs): + workchain = kwargs.get("workchain", -1) + subwalletDefault = 698983191 + workchain # 0x29A9A317 + workchain + subwallet = kwargs.get("subwallet", subwalletDefault) + rewardShare = kwargs.get("rewardShare", 0) + coverAbility = kwargs.get("coverAbility", 0) + self.local.AddLog("start CreateNominationController function", "debug") + walletPath = self.walletsDir + name + contractPath = self.contractsDir + "nomination-contract/" + if not os.path.isdir(contractPath): + self.DownloadContract("https://github.com/EmelyanenkoK/nomination-contract") + #end if + + fiftScript = contractPath + "scripts/new-nomination-controller.fif" + args = [fiftScript, workchain, subwallet, nominatorAddr, rewardShare, coverAbility, walletPath] + result = self.fift.Run(args) + version = "v3r3" + wallet = self.GetLocalWallet(name, version) + self.SetWalletVersion(wallet.addrB64, version) + #end define + + def DepositToNominationController(self, walletName, destAddr, amount): + wallet = self.GetLocalWallet(walletName) + bocPath = self.contractsDir + "nomination-contract/scripts/add-stake.boc" + resultFilePath = self.SignBocWithWallet(wallet, bocPath, destAddr, amount) + self.SendFile(resultFilePath, wallet) + #end define + + def WithdrawFromNominationController(self, walletName, destAddr, amount): + wallet = self.GetLocalWallet(walletName) + fiftScript = self.contractsDir + "nomination-contract/scripts/request-stake.fif" # withdraw-stake.fif + bocPath = self.contractsDir + "nomination-contract/scripts/withdraw-stake" + args = [fiftScript, amount, bocPath] + result = self.fift.Run(args) + bocPath = parse(result, "Saved to file ", ")") + resultFilePath = self.SignBocWithWallet(wallet, bocPath, destAddr, 1) + self.SendFile(resultFilePath, wallet) + #end define + + def SendRequestToNominationController(self, walletName, destAddr): + wallet = self.GetLocalWallet(walletName) + bocPath = self.contractsDir + "nomination-contract/scripts/elector-refund.boc" + resultFilePath = self.SignBocWithWallet(wallet, bocPath, destAddr, 1.5) + self.SendFile(resultFilePath, wallet) + #end define + + def CreateRestrictedWallet(self, name, ownerAddr, **kwargs): + workchain = kwargs.get("workchain", 0) + subwalletDefault = 698983191 + workchain # 0x29A9A317 + workchain + subwallet = kwargs.get("subwallet", subwalletDefault) + self.local.AddLog("start CreateRestrictedWallet function", "debug") + walletPath = self.walletsDir + name + contractPath = self.contractsDir + "nomination-contract/" + if not os.path.isdir(contractPath): + self.DownloadContract("https://github.com/EmelyanenkoK/nomination-contract") + #end if + + fiftScript = contractPath + "scripts/new-restricted-wallet.fif" + args = [fiftScript, workchain, subwallet, ownerAddr, walletPath] + result = self.fift.Run(args) + version = "v3r4" + wallet = self.GetLocalWallet(name, version) + self.SetWalletVersion(wallet.addrB64, version) + #end define + def CreatePool(self, poolName, validatorRewardSharePercent, maxNominatorsCount, minValidatorStake, minNominatorStake): - local.add_log("start CreatePool function", "debug") + self.local.add_log("start CreatePool function", "debug") validatorRewardShare = int(validatorRewardSharePercent * 100) contractPath = self.contractsDir + "nominator-pool/" if not os.path.isdir(contractPath): @@ -3421,7 +3302,7 @@ def CreatePool(self, poolName, validatorRewardSharePercent, maxNominatorsCount, filePath = self.poolsDir + poolName if os.path.isfile(filePath + ".addr"): - local.add_log("CreatePool warning: Pool already exists: " + filePath, "warning") + self.local.add_log("CreatePool warning: Pool already exists: " + filePath, "warning") return #end if @@ -3443,7 +3324,7 @@ def CreatePool(self, poolName, validatorRewardSharePercent, maxNominatorsCount, #end define def ActivatePool(self, pool, ex=True): - local.add_log("start ActivatePool function", "debug") + self.local.add_log("start ActivatePool function", "debug") for i in range(10): time.sleep(3) account = self.GetAccount(pool.addrB64) @@ -3456,7 +3337,7 @@ def ActivatePool(self, pool, ex=True): def DepositToPool(self, walletName, poolAddr, amount): wallet = self.GetLocalWallet(walletName) - bocPath = local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" + bocPath = self.local.buffer.my_temp_dir + wallet.name + "validator-deposit-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/validator-deposit.fif" args = [fiftScript, bocPath] result = self.fift.Run(args) @@ -3473,9 +3354,9 @@ def WithdrawFromPool(self, poolAddr, amount): #end define def WithdrawFromPoolProcess(self, poolAddr, amount): - local.add_log("start WithdrawFromPoolProcess function", "debug") - wallet = self.GetValidatorWallet() - bocPath = local.buffer.my_temp_dir + wallet.name + "validator-withdraw-query.boc" + self.local.add_log("start WithdrawFromPoolProcess function", "debug") + wallet = self.GetLocalWallet() + bocPath = self.local.buffer.my_temp_dir + wallet.name + "validator-withdraw-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/validator-withdraw.fif" args = [fiftScript, amount, bocPath] result = self.fift.Run(args) @@ -3484,10 +3365,10 @@ def WithdrawFromPoolProcess(self, poolAddr, amount): #end define def PendWithdrawFromPool(self, poolAddr, amount): - local.add_log("start PendWithdrawFromPool function", "debug") + self.local.add_log("start PendWithdrawFromPool function", "debug") pendingWithdraws = self.GetPendingWithdraws() pendingWithdraws[poolAddr] = amount - local.save() + self.local.save() #end define def HandlePendingWithdraw(self, pendingWithdraws, poolAddr): @@ -3498,15 +3379,15 @@ def HandlePendingWithdraw(self, pendingWithdraws, poolAddr): def GetPendingWithdraws(self): bname = "pendingWithdraws" - pendingWithdraws = local.db.get(bname) + pendingWithdraws = self.local.db.get(bname) if pendingWithdraws is None: pendingWithdraws = dict() - local.db[bname] = pendingWithdraws + self.local.db[bname] = pendingWithdraws return pendingWithdraws #end define def SignElectionRequestWithPoolWithValidator(self, pool, startWorkTime, adnlAddr, validatorPubkey_b64, validatorSignature, maxFactor, stake): - local.add_log("start SignElectionRequestWithPoolWithValidator function", "debug") + self.local.add_log("start SignElectionRequestWithPoolWithValidator function", "debug") fileName = self.tempDir + str(startWorkTime) + "_validator-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/validator-elect-signed.fif" args = [fiftScript, pool.addrB64, startWorkTime, maxFactor, adnlAddr, validatorPubkey_b64, validatorSignature, fileName, stake] @@ -3517,7 +3398,7 @@ def SignElectionRequestWithPoolWithValidator(self, pool, startWorkTime, adnlAddr #end define def PoolProcessRecoverStake(self): - local.add_log("start PoolProcessRecoverStake function", "debug") + self.local.add_log("start PoolProcessRecoverStake function", "debug") resultFilePath = self.tempDir + "recover-query.boc" fiftScript = self.contractsDir + "nominator-pool/func/recover-stake.fif" args = [fiftScript, resultFilePath] @@ -3526,8 +3407,38 @@ def PoolProcessRecoverStake(self): return resultFilePath #end define + def GetControllerData(self, addrB64): + self.local.add_log("start GetControllerData function", "debug") + account = self.GetAccount(addrB64) + if account.status != "active": + return + cmd = "runmethodfull {addrB64} all_data".format(addrB64=addrB64) + result = self.liteClient.Run(cmd) + data = self.Result2List(result) + controllerData = dict() + wallet_data = dict() + wallet_data["seqno"] = data[0][0] + wallet_data["subwallet_id"] = data[0][1] + wallet_data["controller_pubkey"] = data[0][2] + wallet_data["last_used"] = data[0][3] + static_data = dict() + static_data["nominator_address"] = data[1][0] + static_data["controller_reward_share"] = data[1][1] + static_data["controller_cover_ability"] = data[1][2] + balances = dict() + balances["nominator_total_balance"] = data[2][0] + balances["nominator_elector_balance"] = data[2][1] + balances["nominator_withdrawal_request"] = data[2][2] + balances["total_stake_on_elector"] = data[2][3] + controllerData["wallet_data"] = wallet_data + controllerData["static_data"] = static_data + controllerData["balances"] = balances + controllerData["last_sent_stake_time"] = data[3] + return controllerData + #end define + def GetLocalPool(self, poolName): - local.add_log("start GetLocalPool function", "debug") + self.local.add_log("start GetLocalPool function", "debug") if poolName is None: return None filePath = self.poolsDir + poolName @@ -3543,7 +3454,7 @@ def GetLocalPool(self, poolName): #end define def GetPoolsNameList(self): - local.add_log("start GetPoolsNameList function", "debug") + self.local.add_log("start GetPoolsNameList function", "debug") poolsNameList = list() for fileName in os.listdir(self.poolsDir): if fileName.endswith(".addr"): @@ -3554,7 +3465,7 @@ def GetPoolsNameList(self): #end define def GetPools(self): - local.add_log("start GetPools function", "debug") + self.local.add_log("start GetPools function", "debug") pools = list() poolsNameList = self.GetPoolsNameList() for poolName in poolsNameList: @@ -3595,7 +3506,7 @@ def IsPoolReadyToVote(self, addrB64): #end define def GetPoolData(self, addrB64): - local.add_log("start GetPoolData function", "debug") + self.local.add_log("start GetPoolData function", "debug") cmd = f"runmethodfull {addrB64} get_pool_data" result = self.liteClient.Run(cmd) data = self.Result2List(result) @@ -3640,7 +3551,7 @@ def GetNetworkName(self): def GetFunctionBuffer(self, name, timeout=10): timestamp = get_timestamp() - buff = local.buffer.get(name) + buff = self.local.buffer.get(name) if buff is None: return buffTime = buff.get("time") @@ -3655,7 +3566,7 @@ def SetFunctionBuffer(self, name, data): buff = dict() buff["time"] = get_timestamp() buff["data"] = data - local.buffer[name] = buff + self.local.buffer[name] = buff #end define def IsTestnet(self): @@ -3698,523 +3609,6 @@ def IsHash(self, inputHash): #end define #end class -def ng2g(ng): - if ng is None: - return - return int(ng)/10**9 -#end define - -def Init(): - # Event reaction - if ("-e" in sys.argv): - x = sys.argv.index("-e") - eventName = sys.argv[x+1] - Event(eventName) - #end if - - local.run() - - # statistics - local.buffer.blocksData = dict() - local.buffer.transData = dict() - local.buffer.network = [None]*15*6 - local.buffer.diskio = [None]*15*6 - - # scan blocks - local.buffer.masterBlocksList = list() - local.buffer.prevShardsBlock = dict() - local.buffer.blocksNum = 0 - local.buffer.transNum = 0 -#end define - -def Event(eventName): - if eventName == "enableVC": - EnableVcEvent() - elif eventName == "validator down": - ValidatorDownEvent() - local.exit() -#end define - -def EnableVcEvent(): - local.add_log("start EnableVcEvent function", "debug") - # Создать новый кошелек для валидатора - ton = MyTonCore() - wallet = ton.CreateWallet("validator_wallet_001", -1) - local.db["validatorWalletName"] = wallet.name - - # Создать новый ADNL адрес для валидатора - adnlAddr = ton.CreateNewKey() - ton.AddAdnlAddrToValidator(adnlAddr) - local.db["adnlAddr"] = adnlAddr - - # Сохранить - local.save() -#end define - -def ValidatorDownEvent(): - local.add_log("start ValidatorDownEvent function", "debug") - local.add_log("Validator is down", "error") -#end define - -def Elections(ton): - usePool = local.db.get("usePool") - if usePool == True: - ton.PoolsUpdateValidatorSet() - ton.RecoverStake() - ton.ElectionEntry() - else: - ton.RecoverStake() - ton.ElectionEntry() -#end define - -def Statistics(): - ReadNetworkData() - SaveNetworkStatistics() - #ReadTransData(scanner) - SaveTransStatistics() - ReadDiskData() - SaveDiskStatistics() -#end define - -def ReadDiskData(): - timestamp = get_timestamp() - disks = GetDisksList() - buff = psutil.disk_io_counters(perdisk=True) - data = dict() - for name in disks: - data[name] = dict() - data[name]["timestamp"] = timestamp - data[name]["busyTime"] = buff[name].busy_time - data[name]["readBytes"] = buff[name].read_bytes - data[name]["writeBytes"] = buff[name].write_bytes - data[name]["readCount"] = buff[name].read_count - data[name]["writeCount"] = buff[name].write_count - #end for - - local.buffer.diskio.pop(0) - local.buffer.diskio.append(data) -#end define - -def SaveDiskStatistics(): - data = local.buffer.diskio - data = data[::-1] - zerodata = data[0] - buff1 = data[1*6-1] - buff5 = data[5*6-1] - buff15 = data[15*6-1] - if buff5 is None: - buff5 = buff1 - if buff15 is None: - buff15 = buff5 - #end if - - disksLoadAvg = dict() - disksLoadPercentAvg = dict() - iopsAvg = dict() - disks = GetDisksList() - for name in disks: - if zerodata[name]["busyTime"] == 0: - continue - diskLoad1, diskLoadPercent1, iops1 = CalculateDiskStatistics(zerodata, buff1, name) - diskLoad5, diskLoadPercent5, iops5 = CalculateDiskStatistics(zerodata, buff5, name) - diskLoad15, diskLoadPercent15, iops15 = CalculateDiskStatistics(zerodata, buff15, name) - disksLoadAvg[name] = [diskLoad1, diskLoad5, diskLoad15] - disksLoadPercentAvg[name] = [diskLoadPercent1, diskLoadPercent5, diskLoadPercent15] - iopsAvg[name] = [iops1, iops5, iops15] - #end fore - - # save statistics - statistics = local.db.get("statistics", dict()) - statistics["disksLoadAvg"] = disksLoadAvg - statistics["disksLoadPercentAvg"] = disksLoadPercentAvg - statistics["iopsAvg"] = iopsAvg - local.db["statistics"] = statistics -#end define - -def CalculateDiskStatistics(zerodata, data, name): - if data is None: - return None, None, None - data = data[name] - zerodata = zerodata[name] - timeDiff = zerodata["timestamp"] - data["timestamp"] - busyTimeDiff = zerodata["busyTime"] - data["busyTime"] - diskReadDiff = zerodata["readBytes"] - data["readBytes"] - diskWriteDiff = zerodata["writeBytes"] - data["writeBytes"] - diskReadCountDiff = zerodata["readCount"] - data["readCount"] - diskWriteCountDiff = zerodata["writeCount"] - data["writeCount"] - diskLoadPercent = busyTimeDiff /1000 /timeDiff *100 # /1000 - to second, *100 - to percent - diskLoadPercent = round(diskLoadPercent, 2) - diskRead = diskReadDiff /timeDiff - diskWrite = diskWriteDiff /timeDiff - diskReadCount = diskReadCountDiff /timeDiff - diskWriteCount = diskWriteCountDiff /timeDiff - diskLoad = b2mb(diskRead + diskWrite) - iops = round(diskReadCount + diskWriteCount, 2) - return diskLoad, diskLoadPercent, iops -#end define - -def GetDisksList(): - data = list() - buff = os.listdir("/sys/block/") - for item in buff: - if "loop" in item: - continue - data.append(item) - #end for - data.sort() - return data -#end define - -def ReadNetworkData(): - timestamp = get_timestamp() - interfaceName = get_internet_interface_name() - buff = psutil.net_io_counters(pernic=True) - buff = buff[interfaceName] - data = dict() - data["timestamp"] = timestamp - data["bytesRecv"] = buff.bytes_recv - data["bytesSent"] = buff.bytes_sent - data["packetsSent"] = buff.packets_sent - data["packetsRecv"] = buff.packets_recv - - local.buffer.network.pop(0) - local.buffer.network.append(data) -#end define - -def SaveNetworkStatistics(): - data = local.buffer.network - data = data[::-1] - zerodata = data[0] - buff1 = data[1*6-1] - buff5 = data[5*6-1] - buff15 = data[15*6-1] - if buff5 is None: - buff5 = buff1 - if buff15 is None: - buff15 = buff5 - #end if - - netLoadAvg = dict() - ppsAvg = dict() - networkLoadAvg1, ppsAvg1 = CalculateNetworkStatistics(zerodata, buff1) - networkLoadAvg5, ppsAvg5 = CalculateNetworkStatistics(zerodata, buff5) - networkLoadAvg15, ppsAvg15 = CalculateNetworkStatistics(zerodata, buff15) - netLoadAvg = [networkLoadAvg1, networkLoadAvg5, networkLoadAvg15] - ppsAvg = [ppsAvg1, ppsAvg5, ppsAvg15] - - # save statistics - statistics = local.db.get("statistics", dict()) - statistics["netLoadAvg"] = netLoadAvg - statistics["ppsAvg"] = ppsAvg - local.db["statistics"] = statistics -#end define - -def CalculateNetworkStatistics(zerodata, data): - if data is None: - return None, None - timeDiff = zerodata["timestamp"] - data["timestamp"] - bytesRecvDiff = zerodata["bytesRecv"] - data["bytesRecv"] - bytesSentDiff = zerodata["bytesSent"] - data["bytesSent"] - packetsRecvDiff = zerodata["packetsRecv"] - data["packetsRecv"] - packetsSentDiff = zerodata["packetsSent"] - data["packetsSent"] - bitesRecvAvg = bytesRecvDiff /timeDiff *8 - bitesSentAvg = bytesSentDiff /timeDiff *8 - packetsRecvAvg = packetsRecvDiff /timeDiff - packetsSentAvg = packetsSentDiff /timeDiff - netLoadAvg = b2mb(bitesRecvAvg + bitesSentAvg) - ppsAvg = round(packetsRecvAvg + packetsSentAvg, 2) - return netLoadAvg, ppsAvg -#end define - -def ReadTransData(scanner): - transData = local.buffer.transData - SetToTimeData(transData, scanner.transNum) - ShortTimeData(transData) -#end define - -def SetToTimeData(timeDataList, data): - timenow = int(time.time()) - timeDataList[timenow] = data -#end define - -def ShortTimeData(data, max=120, diff=20): - if len(data) < max: - return - buff = data.copy() - data.clear() - keys = sorted(buff.keys(), reverse=True) - for item in keys[:max-diff]: - data[item] = buff[item] -#end define - -def SaveTransStatistics(): - tps1 = GetTps(60) - tps5 = GetTps(60*5) - tps15 = GetTps(60*15) - - # save statistics - statistics = local.db.get("statistics", dict()) - statistics["tpsAvg"] = [tps1, tps5, tps15] - local.db["statistics"] = statistics -#end define - -def GetDataPerSecond(data, timediff): - if len(data) == 0: - return - timenow = sorted(data.keys())[-1] - now = data.get(timenow) - prev = GetItemFromTimeData(data, timenow-timediff) - if prev is None: - return - diff = now - prev - result = diff / timediff - result = round(result, 2) - return result -#end define - -def GetItemFromTimeData(data, timeneed): - if timeneed in data: - result = data.get(timeneed) - else: - result = data[min(data.keys(), key=lambda k: abs(k-timeneed))] - return result -#end define - - -def GetTps(timediff): - data = local.buffer.transData - tps = GetDataPerSecond(data, timediff) - return tps -#end define - -def GetBps(timediff): - data = local.buffer.blocksData - bps = GetDataPerSecond(data, timediff) - return bps -#end define - -def GetBlockTimeAvg(timediff): - bps = GetBps(timediff) - if bps is None or bps == 0: - return - result = 1/bps - result = round(result, 2) - return result -#end define - -def Offers(ton): - saveOffers = ton.GetSaveOffers() - offers = ton.GetOffers() - for offer in offers: - offerHash = offer.get("hash") - offerPseudohash = offer.get("pseudohash") - saveOfferPseudohash = saveOffers.get(offerHash) - if offerPseudohash == saveOfferPseudohash: - ton.VoteOffer(offerHash) -#end define - -def Domains(ton): - pass -#end define - -def GetUname(): - data = os.uname() - result = dict(zip('sysname nodename release version machine'.split(), data)) - result.pop("nodename") - return result -#end define - -def GetMemoryInfo(): - result = dict() - data = psutil.virtual_memory() - result["total"] = round(data.total / 10**9, 2) - result["usage"] = round(data.used / 10**9, 2) - result["usagePercent"] = data.percent - return result -#end define - -def GetSwapInfo(): - result = dict() - data = psutil.swap_memory() - result["total"] = round(data.total / 10**9, 2) - result["usage"] = round(data.used / 10**9, 2) - result["usagePercent"] = data.percent - return result -#end define - -def GetValidatorProcessInfo(): - pid = get_service_pid("validator") - if pid == None or pid == 0: - return - p = psutil.Process(pid) - mem = p.memory_info() - result = dict() - result["cpuPercent"] = p.cpu_percent() - memory = dict() - memory["rss"] = mem.rss - memory["vms"] = mem.vms - memory["shared"] = mem.shared - memory["text"] = mem.text - memory["lib"] = mem.lib - memory["data"] = mem.data - memory["dirty"] = mem.dirty - result["memory"] = memory - #io = p.io_counters() # Permission denied: '/proc/{pid}/io' - return result -#end define - -def Telemetry(ton): - sendTelemetry = local.db.get("sendTelemetry") - if sendTelemetry is not True: - return - #end if - - # Get validator status - data = dict() - data["adnlAddr"] = ton.GetAdnlAddr() - data["validatorStatus"] = ton.GetValidatorStatus() - data["cpuNumber"] = psutil.cpu_count() - data["cpuLoad"] = get_load_avg() - data["netLoad"] = ton.GetStatistics("netLoadAvg") - data["tps"] = ton.GetStatistics("tpsAvg") - data["disksLoad"] = ton.GetStatistics("disksLoadAvg") - data["disksLoadPercent"] = ton.GetStatistics("disksLoadPercentAvg") - data["iops"] = ton.GetStatistics("iopsAvg") - data["pps"] = ton.GetStatistics("ppsAvg") - data["dbUsage"] = ton.GetDbUsage() - data["memory"] = GetMemoryInfo() - data["swap"] = GetSwapInfo() - data["uname"] = GetUname() - data["vprocess"] = GetValidatorProcessInfo() - elections = local.try_function(ton.GetElectionEntries) - complaints = local.try_function(ton.GetComplaints) - - # Get git hashes - gitHashes = dict() - gitHashes["mytonctrl"] = get_git_hash("/usr/src/mytonctrl") - gitHashes["validator"] = GetBinGitHash("/usr/bin/ton/validator-engine/validator-engine") - data["gitHashes"] = gitHashes - data["stake"] = local.db.get("stake") - - # Get validator config - vconfig = ton.GetValidatorConfig() - data["fullnode_adnl"] = vconfig.fullnode - - # Send data to toncenter server - liteUrl_default = "https://telemetry.toncenter.com/report_status" - liteUrl = local.db.get("telemetryLiteUrl", liteUrl_default) - output = json.dumps(data) - resp = requests.post(liteUrl, data=output, timeout=3) -#end define - -def GetBinGitHash(path, short=False): - if not os.path.isfile(path): - return - args = [path, "--version"] - process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3) - output = process.stdout.decode("utf-8") - if "build information" not in output: - return - buff = output.split(' ') - start = buff.index("Commit:") + 1 - result = buff[start].replace(',', '') - if short is True: - result = result[:7] - return result -#end define - -def OverlayTelemetry(ton): - sendTelemetry = local.db.get("sendTelemetry") - if sendTelemetry is not True: - return - #end if - - # Get validator status - data = dict() - data["adnlAddr"] = ton.GetAdnlAddr() - data["overlaysStats"] = ton.GetOverlaysStats() - - # Send data to toncenter server - overlayUrl_default = "https://telemetry.toncenter.com/report_overlays" - overlayUrl = local.db.get("overlayTelemetryUrl", overlayUrl_default) - output = json.dumps(data) - resp = requests.post(overlayUrl, data=output, timeout=3) -#end define - -def Complaints(ton): - validatorIndex = ton.GetValidatorIndex() - if validatorIndex < 0: - return - #end if - - # Voting for complaints - config32 = ton.GetConfig32() - electionId = config32.get("startWorkTime") - complaintsHashes = ton.SaveComplaints(electionId) - complaints = ton.GetComplaints(electionId) - for key, item in complaints.items(): - complaintHash = item.get("hash") - complaintHash_hex = Dec2HexAddr(complaintHash) - if complaintHash_hex in complaintsHashes: - ton.VoteComplaint(electionId, complaintHash) -#end define - -def Slashing(ton): - isSlashing = local.db.get("isSlashing") - if isSlashing is not True: - return - #end if - - # Creating complaints - slash_time = local.buffer.slash_time - config32 = ton.GetConfig32() - start = config32.get("startWorkTime") - end = config32.get("endWorkTime") - local.add_log("slash_time {}, start {}, end {}".format(slash_time, start, end), "debug") - if slash_time != start: - end -= 60 - ton.CheckValidators(start, end) - local.buffer.slash_time = start -#end define - -def ScanLiteServers(ton): - # Считать список серверов - filePath = ton.liteClient.configPath - file = open(filePath, 'rt') - text = file.read() - file.close() - data = json.loads(text) - - # Пройтись по серверам - result = list() - liteservers = data.get("liteservers") - for index in range(len(liteservers)): - try: - ton.liteClient.Run("last", index=index) - result.append(index) - except: pass - #end for - - # Записать данные в базу - local.db["liteServers"] = result -#end define - -def General(): - local.add_log("start General function", "debug") - ton = MyTonCore() - - # Запустить потоки - local.start_cycle(Elections, sec=600, args=(ton, )) - local.start_cycle(Statistics, sec=10) - local.start_cycle(Offers, sec=600, args=(ton, )) - local.start_cycle(Complaints, sec=600, args=(ton, )) - local.start_cycle(Slashing, sec=600, args=(ton, )) - local.start_cycle(Domains, sec=600, args=(ton, )) - local.start_cycle(Telemetry, sec=60, args=(ton, )) - local.start_cycle(OverlayTelemetry, sec=7200, args=(ton, )) - local.start_cycle(ScanLiteServers, sec=60, args=(ton,)) - thr_sleep() -#end define def Dec2HexAddr(dec): h = dec2hex(dec) @@ -4222,31 +3616,3 @@ def Dec2HexAddr(dec): h64 = hu.rjust(64, "0") return h64 #end define - -def xhex2hex(x): - try: - b = x[1:] - h = b.lower() - return h - except: - return None -#end define - -def hex2base64(h): - b = bytes.fromhex(h) - b64 = base64.b64encode(b) - s = b64.decode("utf-8") - return s -#end define - - - - -### -### Start of the program -### - -if __name__ == "__main__": - Init() - General() -#end if diff --git a/mytoncore/tonblocksscanner.py b/mytoncore/tonblocksscanner.py new file mode 100644 index 00000000..6f607789 --- /dev/null +++ b/mytoncore/tonblocksscanner.py @@ -0,0 +1,208 @@ +import threading +import time + +from mytoncore.models import Block + + +class TonBlocksScanner(): + def __init__(self, ton, **kwargs): + self.ton = ton + self.prevMasterBlock = None + self.prevShardsBlock = dict() + self.blocksNum = 0 + self.transNum = 0 + self.nbr = kwargs.get("nbr") #NewBlockReaction + self.ntr = kwargs.get("ntr") #NewTransReaction + self.nmr = kwargs.get("nmr") #NewMessageReaction + self.local = kwargs.get("local") + self.sync = kwargs.get("sync", False) + self.delay = 0 + self.working = False + self.closing = False + #end define + + def Run(self): + self.StartThread(self.ScanBlocks, args=()) + self.StartThread(self.ThreadBalancing, args=()) + self.StartThread(self.StatusReading, args=()) + #end define + + def StartThread(self, func, args): + threading.Thread(target=func, args=args, name=func.__name__, daemon=True).start() + #end define + + def StartWithMode(self, func, args): + if self.sync: + func(*args) + else: + self.StartThread(func, args) + #end define + + def AddLog(self, text, type): + if self.local: + self.local.AddLog(text, type) + else: + print(text) + #end define + + def Try(self, func, **kwargs): + args = kwargs.get("args", tuple()) + for step in range(10): + time.sleep(step) + try: + result = func(*args) + return result + except Exception as ex: + err = ex + text = f"{func.__name__} step: {step}, error: {err}" + self.AddLog(text, "error") + raise Exception(err) + #end define + + def SetStartBlock(self, workchain, shardchain, seqno): + workchainType = type(workchain) + shardchainType = type(shardchain) + seqnoType = type(seqno) + if workchainType != int: + raise Exception(f"SetStartBlock error: workchain type mast be int, not {workchainType}") + if shardchainType != str: + raise Exception(f"SetStartBlock error: shardchain type mast be str, not {shardchainType}") + if seqnoType != int: + raise Exception(f"SetStartBlock error: seqno type mast be int, not {seqnoType}") + #end if + + block = Block() + block.workchain = workchain + block.shardchain = shardchain + block.seqno = seqno + if workchain == -1: + self.prevMasterBlock = block + else: + self.SetShardPrevBlock(block) + self.sync = True + #end define + + def ThreadBalancing(self): + while True: + tnum = threading.active_count() + if tnum > 100: + self.delay += 0.1 + elif tnum > 50: + self.delay += 0.01 + elif tnum < 50: + self.delay -= 0.1 + elif tnum < 100: + self.delay -= 0.01 + if self.delay < 0: + self.delay = 0 + if self.closing is True: + exit() + time.sleep(0.1) + #end define + + def StatusReading(self): + while True: + validatorStatus = self.ton.GetValidatorStatus() + validatorOutOfSync = validatorStatus.get("outOfSync") + if self.ton.liteClient.pubkeyPath is None: + self.working = False + self.closing = True + text = "TonBlocksScanner error: local liteserver is not configured, stop thread." + self.AddLog(text, "error") + exit() + if validatorOutOfSync > 20: + self.working = False + text = f"TonBlocksScanner warning: local liteserver is out of sync: {validatorOutOfSync}." + self.AddLog(text, "warning") + else: + self.working = True + time.sleep(10) + #end define + + def ScanBlocks(self): + while True: + if self.working is True: + self.ScanBlock() + if self.closing is True: + exit() + time.sleep(1) + #end define + + def ScanBlock(self): + block = self.Try(self.ton.GetLastBlock) + self.StartThread(self.SearchMissBlocks, args=(block, self.prevMasterBlock)) + if block != self.prevMasterBlock: + self.StartWithMode(self.ReadBlock, args=(block,)) + self.prevMasterBlock = block + #end define + + def ReadBlock(self, block): + self.StartWithMode(self.NewBlockReaction, args=(block,)) + shards = self.Try(self.ton.GetShards, args=(block,)) + for shard in shards: + self.StartThread(self.ReadShard, args=(shard,)) + #end define + + def ReadShard(self, shard): + block = shard.get("block") + prevBlock = self.GetShardPrevBlock(block.shardchain) + self.StartThread(self.SearchMissBlocks, args=(block, prevBlock)) + if block != prevBlock: + self.StartWithMode(self.NewBlockReaction, args=(block,)) + self.SetShardPrevBlock(block) + #end define + + def SearchMissBlocks(self, block, prevBlock): + if prevBlock is None: + return + diff = block.seqno - prevBlock.seqno + #for i in range(1, diff): + for i in range(diff-1, 0, -1): + workchain = block.workchain + shardchain = block.shardchain + seqno = block.seqno - i + self.StartWithMode(self.SearchBlock, args=(workchain, shardchain, seqno)) + #end define + + def SearchBlock(self, workchain, shardchain, seqno): + if self.delay != 0: + time.sleep(self.delay) + block = self.Try(self.ton.GetBlock, args=(workchain, shardchain, seqno)) + self.StartWithMode(self.NewBlockReaction, args=(block,)) + #end define + + def GetShardPrevBlock(self, shardchain): + prevBlock = self.prevShardsBlock.get(shardchain) + return prevBlock + #end define + + def SetShardPrevBlock(self, prevBlock): + self.prevShardsBlock[prevBlock.shardchain] = prevBlock + #end define + + def NewBlockReaction(self, block): + #print(f"{bcolors.green} block: {bcolors.endc} {block}") + self.blocksNum += 1 + if self.nbr: + self.StartThread(self.nbr, args=(block,)) + transactions = self.Try(self.ton.GetTransactions, args=(block,)) + for trans in transactions: + self.StartWithMode(self.NewTransReaction, args=(trans,)) + #end define + + def NewTransReaction(self, trans): + #print(f"{bcolors.magenta} trans: {bcolors.endc} {self.transNum}", "debug") + self.transNum += 1 + if self.ntr: + self.StartThread(self.ntr, args=(trans,)) + messageList = self.Try(self.ton.GetTrans, args=(trans,)) + for message in messageList: + self.NewMessageReaction(message) + #end define + + def NewMessageReaction(self, message): + if self.nmr: + self.StartThread(self.nmr, args=(message,)) + #print(f"{bcolors.yellow} message: {bcolors.endc} {message}") + #end define +#end class diff --git a/mytoncore/utils.py b/mytoncore/utils.py new file mode 100644 index 00000000..9743b0fc --- /dev/null +++ b/mytoncore/utils.py @@ -0,0 +1,79 @@ +import base64 +import json + + +def str2b64(s): + b = s.encode("utf-8") + b64 = base64.b64encode(b) + b64 = b64.decode("utf-8") + return b64 +# end define + + +def b642str(b64): + b64 = b64.encode("utf-8") + b = base64.b64decode(b64) + s = b.decode("utf-8") + return s +# end define + + +def dict2b64(d): + s = json.dumps(d) + b64 = str2b64(s) + return b64 +# end define + + +def b642dict(b64): + s = b642str(b64) + d = json.loads(s) + return d +# end define + + +def hex2b64(input): # TODO: remove duplicates + hexBytes = bytes.fromhex(input) + b64Bytes = base64.b64encode(hexBytes) + b64String = b64Bytes.decode() + return b64String +# end define + + +def b642hex(input): + b64Bytes = input.encode() + hexBytes = base64.b64decode(b64Bytes) + hexString = hexBytes.hex() + return hexString +# end define + + +def xhex2hex(x): + try: + b = x[1:] + h = b.lower() + return h + except: + return None +#end define + +def hex2base64(h): # TODO: remove duplicates + b = bytes.fromhex(h) + b64 = base64.b64encode(b) + s = b64.decode("utf-8") + return s +#end define + + +def str2bool(str): + if str == "true": + return True + return False +# end define + + +def ng2g(ng): + if ng is None: + return + return int(ng)/10**9 +#end define diff --git a/mytoncore/validator_console.py b/mytoncore/validator_console.py new file mode 100644 index 00000000..85d44891 --- /dev/null +++ b/mytoncore/validator_console.py @@ -0,0 +1,27 @@ +import subprocess + + +class ValidatorConsole: + def __init__(self, local): + self.local = local + self.appPath = None + self.privKeyPath = None + self.pubKeyPath = None + self.addr = None + #end define + + def Run(self, cmd, **kwargs): + console_timeout = self.local.db.console_timeout if self.local.db.console_timeout else 3 + timeout = kwargs.get("timeout", console_timeout) + if self.appPath is None or self.privKeyPath is None or self.pubKeyPath is None: + raise Exception("ValidatorConsole error: Validator console is not settings") + args = [self.appPath, "-k", self.privKeyPath, "-p", self.pubKeyPath, "-a", self.addr, "-v", "0", "--cmd", cmd] + process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + output = process.stdout.decode("utf-8") + err = process.stderr.decode("utf-8") + if len(err) > 0: + self.local.add_log("args: {args}".format(args=args), "error") + raise Exception("ValidatorConsole error: {err}".format(err=err)) + return output + #end define +#end class diff --git a/mytonctrl/__init__.py b/mytonctrl/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mytonctrl/__main__.py b/mytonctrl/__main__.py new file mode 100644 index 00000000..22f2c85b --- /dev/null +++ b/mytonctrl/__main__.py @@ -0,0 +1,4 @@ +from mytonctrl.mytonctrl import mytonctrl + +if __name__ == '__main__': + mytonctrl() diff --git a/mytonctrl.py b/mytonctrl/mytonctrl.py similarity index 69% rename from mytonctrl.py rename to mytonctrl/mytonctrl.py index 09a91834..8fb1542c 100755 --- a/mytonctrl.py +++ b/mytonctrl/mytonctrl.py @@ -1,75 +1,156 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*- +import subprocess +import json +import psutil +import inspect +import pkg_resources +import socket + +from shutil import copyfile +from functools import partial + +from mypylib.mypylib import ( + # GetGitAuthorAndRepo, + # GetGitBranch, + # GetGitHash, + # CheckGitUpdate, + # GetServiceStatus, + # GetServiceUptime, + # GetLoadAvg, + # RunAsRoot, + # time2human, + # timeago, + # Timestamp2Datetime, + # GetTimestamp, + # PrintTable, + # ColorPrint, + # ColorText, + # bcolors, + # MyPyClass, + + int2ip, + get_git_author_and_repo, + get_git_branch, + get_git_hash, + check_git_update, + get_service_status, + get_service_uptime, + get_load_avg, + run_as_root, + time2human, + timeago, + timestamp2datetime, + get_timestamp, + print_table, + color_print, + color_text, + bcolors, + MyPyClass +) + +from mypyconsole.mypyconsole import MyPyConsole +from mytoncore.mytoncore import MyTonCore +from mytoncore.functions import ( + Slashing, + Elections, + GetMemoryInfo, + GetSwapInfo, + GetBinGitHash, +) -from mypylib.mypylib import * -from mypyconsole.mypyconsole import * -from mytoncore import * import sys, getopt, os -local = MyPyClass(__file__) -console = MyPyConsole() -ton = MyTonCore() -def Init(argv): +def Init(local, ton, console, argv): # Load translate table - local.init_translator(local.buffer.my_dir + "translate.json") + translate_path = pkg_resources.resource_filename('mytonctrl', 'resources/translate.json') + local.init_translator(translate_path) + + # this function substitutes local and ton instances if function has this args + def inject_globals(func): + args = [] + for arg_name in inspect.getfullargspec(func)[0]: + if arg_name == 'local': + args.append(local) + elif arg_name == 'ton': + args.append(ton) + return partial(func, *args) # Create user console console.name = "MyTonCtrl" - console.startFunction = PreUp - - console.AddItem("update", Update, local.translate("update_cmd")) - console.AddItem("upgrade", Upgrade, local.translate("upgrade_cmd")) - console.AddItem("installer", Installer, local.translate("installer_cmd")) - console.AddItem("status", PrintStatus, local.translate("status_cmd")) - console.AddItem("seqno", Seqno, local.translate("seqno_cmd")) - console.AddItem("getconfig", GetConfig, local.translate("getconfig_cmd")) - - console.AddItem("nw", CreatNewWallet, local.translate("nw_cmd")) - console.AddItem("aw", ActivateWallet, local.translate("aw_cmd")) - console.AddItem("wl", PrintWalletsList, local.translate("wl_cmd")) - console.AddItem("iw", ImportWallet, local.translate("iw_cmd")) - console.AddItem("swv", SetWalletVersion, local.translate("swv_cmd")) - console.AddItem("ew", ExportWallet, local.translate("ex_cmd")) - console.AddItem("dw", DeleteWallet, local.translate("dw_cmd")) - - console.AddItem("vas", ViewAccountStatus, local.translate("vas_cmd")) - console.AddItem("vah", ViewAccountHistory, local.translate("vah_cmd")) - console.AddItem("mg", MoveCoins, local.translate("mg_cmd")) - console.AddItem("mgtp", MoveCoinsThroughProxy, local.translate("mgtp_cmd")) - - console.AddItem("nb", CreatNewBookmark, local.translate("nb_cmd")) - console.AddItem("bl", PrintBookmarksList, local.translate("bl_cmd")) - console.AddItem("db", DeleteBookmark, local.translate("db_cmd")) - - console.AddItem("nd", NewDomain, local.translate("nd_cmd")) - console.AddItem("dl", PrintDomainsList, local.translate("dl_cmd")) - console.AddItem("vds", ViewDomainStatus, local.translate("vds_cmd")) - console.AddItem("dd", DeleteDomain, local.translate("dd_cmd")) - console.AddItem("gdfa", GetDomainFromAuction, local.translate("gdfa_cmd")) - - console.AddItem("ol", PrintOffersList, local.translate("ol_cmd")) - console.AddItem("vo", VoteOffer, local.translate("vo_cmd")) - console.AddItem("od", OfferDiff, local.translate("od_cmd")) - - console.AddItem("el", PrintElectionEntriesList, local.translate("el_cmd")) - console.AddItem("ve", VoteElectionEntry, local.translate("ve_cmd")) - console.AddItem("vl", PrintValidatorList, local.translate("vl_cmd")) - console.AddItem("cl", PrintComplaintsList, local.translate("cl_cmd")) - console.AddItem("vc", VoteComplaint, local.translate("vc_cmd")) - - console.AddItem("get", GetSettings, local.translate("get_cmd")) - console.AddItem("set", SetSettings, local.translate("set_cmd")) - console.AddItem("xrestart", Xrestart, local.translate("xrestart_cmd")) - console.AddItem("xlist", Xlist, local.translate("xlist_cmd")) - - console.AddItem("new_pool", NewPool, local.translate("new_pool_cmd")) - console.AddItem("pools_list", PrintPoolsList, local.translate("pools_list_cmd")) - console.AddItem("get_pool_data", GetPoolData, local.translate("get_pool_data_cmd")) - console.AddItem("activate_pool", ActivatePool, local.translate("activate_pool_cmd")) - console.AddItem("deposit_to_pool", DepositToPool, local.translate("deposit_to_pool_cmd")) - console.AddItem("withdraw_from_pool", WithdrawFromPool, local.translate("withdraw_from_pool_cmd")) - console.AddItem("delete_pool", DeletePool, local.translate("delete_pool_cmd")) + console.startFunction = inject_globals(PreUp) + + console.AddItem("update", inject_globals(Update), local.translate("update_cmd")) + console.AddItem("upgrade", inject_globals(Upgrade), local.translate("upgrade_cmd")) + console.AddItem("installer", inject_globals(Installer), local.translate("installer_cmd")) + console.AddItem("status", inject_globals(PrintStatus), local.translate("status_cmd")) + console.AddItem("seqno", inject_globals(Seqno), local.translate("seqno_cmd")) + console.AddItem("getconfig", inject_globals(GetConfig), local.translate("getconfig_cmd")) + + console.AddItem("nw", inject_globals(CreatNewWallet), local.translate("nw_cmd")) + console.AddItem("aw", inject_globals(ActivateWallet), local.translate("aw_cmd")) + console.AddItem("wl", inject_globals(PrintWalletsList), local.translate("wl_cmd")) + console.AddItem("iw", inject_globals(ImportWallet), local.translate("iw_cmd")) + console.AddItem("swv", inject_globals(SetWalletVersion), local.translate("swv_cmd")) + console.AddItem("ew", inject_globals(ExportWallet), local.translate("ex_cmd")) + console.AddItem("dw", inject_globals(DeleteWallet), local.translate("dw_cmd")) + + console.AddItem("vas", inject_globals(ViewAccountStatus), local.translate("vas_cmd")) + console.AddItem("vah", inject_globals(ViewAccountHistory), local.translate("vah_cmd")) + console.AddItem("mg", inject_globals(MoveCoins), local.translate("mg_cmd")) + console.AddItem("mgtp", inject_globals(MoveCoinsThroughProxy), local.translate("mgtp_cmd")) + + console.AddItem("nb", inject_globals(CreatNewBookmark), local.translate("nb_cmd")) + console.AddItem("bl", inject_globals(PrintBookmarksList), local.translate("bl_cmd")) + console.AddItem("db", inject_globals(DeleteBookmark), local.translate("db_cmd")) + + # console.AddItem("nr", inject_globals(CreatNewAutoTransferRule), local.translate("nr_cmd")) # "Добавить правило автопереводов в расписание / Create new auto transfer rule" + # console.AddItem("rl", inject_globals(PrintAutoTransferRulesList), local.translate("rl_cmd")) # "Показать правила автопереводов / Show auto transfer rule list" + # console.AddItem("dr", inject_globals(DeleteAutoTransferRule), local.translate("dr_cmd")) # "Удалить правило автопереводов из расписания / Delete auto transfer rule" + + console.AddItem("nd", inject_globals(NewDomain), local.translate("nd_cmd")) + console.AddItem("dl", inject_globals(PrintDomainsList), local.translate("dl_cmd")) + console.AddItem("vds", inject_globals(ViewDomainStatus), local.translate("vds_cmd")) + console.AddItem("dd", inject_globals(DeleteDomain), local.translate("dd_cmd")) + console.AddItem("gdfa", inject_globals(GetDomainFromAuction), local.translate("gdfa_cmd")) + + console.AddItem("ol", inject_globals(PrintOffersList), local.translate("ol_cmd")) + console.AddItem("vo", inject_globals(VoteOffer), local.translate("vo_cmd")) + console.AddItem("od", inject_globals(OfferDiff), local.translate("od_cmd")) + + console.AddItem("el", inject_globals(PrintElectionEntriesList), local.translate("el_cmd")) + console.AddItem("ve", inject_globals(VoteElectionEntry), local.translate("ve_cmd")) + console.AddItem("vl", inject_globals(PrintValidatorList), local.translate("vl_cmd")) + console.AddItem("cl", inject_globals(PrintComplaintsList), local.translate("cl_cmd")) + console.AddItem("vc", inject_globals(VoteComplaint), local.translate("vc_cmd")) + + console.AddItem("get", inject_globals(GetSettings), local.translate("get_cmd")) + console.AddItem("set", inject_globals(SetSettings), local.translate("set_cmd")) + console.AddItem("xrestart", inject_globals(Xrestart), local.translate("xrestart_cmd")) + console.AddItem("xlist", inject_globals(Xlist), local.translate("xlist_cmd")) + #console.AddItem("gpk", inject_globals(GetPubKey), local.translate("gpk_cmd")) + #console.AddItem("ssoc", inject_globals(SignShardOverlayCert), local.translate("ssoc_cmd")) + #console.AddItem("isoc", inject_globals(ImportShardOverlayCert), local.translate("isoc_cmd")) + + #console.AddItem("new_nomination_controller", inject_globals(NewNominationController), local.translate("new_controller_cmd")) + #console.AddItem("get_nomination_controller_data", inject_globals(GetNominationControllerData), local.translate("get_nomination_controller_data_cmd")) + #console.AddItem("deposit_to_nomination_controller", inject_globals(DepositToNominationController), local.translate("deposit_to_controller_cmd")) + #console.AddItem("withdraw_from_nomination_controller", inject_globals(WithdrawFromNominationController), local.translate("withdraw_from_nomination_controller_cmd")) + #console.AddItem("request_to_nomination_controller", inject_globals(SendRequestToNominationController), local.translate("request_to_nomination_controller_cmd")) + #console.AddItem("new_restricted_wallet", inject_globals(NewRestrictedWallet), local.translate("new_restricted_wallet_cmd")) + + console.AddItem("new_pool", inject_globals(NewPool), local.translate("new_pool_cmd")) + console.AddItem("pools_list", inject_globals(PrintPoolsList), local.translate("pools_list_cmd")) + console.AddItem("get_pool_data", inject_globals(GetPoolData), local.translate("get_pool_data_cmd")) + console.AddItem("activate_pool", inject_globals(ActivatePool), local.translate("activate_pool_cmd")) + console.AddItem("deposit_to_pool", inject_globals(DepositToPool), local.translate("deposit_to_pool_cmd")) + console.AddItem("withdraw_from_pool", inject_globals(WithdrawFromPool), local.translate("withdraw_from_pool_cmd")) + console.AddItem("delete_pool", inject_globals(DeletePool), local.translate("delete_pool_cmd")) + #console.AddItem("update_validator_set", inject_globals(UpdateValidatorSet), local.translate("update_validator_set_cmd")) + + # console.AddItem("pt", inject_globals(PrintTest), "PrintTest") + # console.AddItem("sl", inject_globals(sl), "sl") # Process input parameters opts, args = getopt.getopt(argv,"hc:w:",["config=","wallets="]) @@ -101,14 +182,15 @@ def Init(argv): local.run() #end define -def PreUp(): - CheckMytonctrlUpdate() - check_vport() +def PreUp(local, ton): + CheckMytonctrlUpdate(local) + check_vport(local, ton) # CheckTonUpdate() #end define def Installer(args): - args = ["python3", "/usr/src/mytonctrl/mytoninstaller.py"] + # args = ["python3", "/usr/src/mytonctrl/mytoninstaller.py"] + args = ["python3", "-m", "mytoninstaller"] subprocess.run(args) #end define @@ -118,7 +200,27 @@ def GetItemFromList(data, index): except: pass #end define -def check_vport(): +def GetAuthorRepoBranchFromArgs(args): + data = dict() + arg1 = GetItemFromList(args, 0) + arg2 = GetItemFromList(args, 1) + if arg1: + if "https://" in arg1: + buff = arg1[8:].split('/') + print(f"buff: {buff}") + data["author"] = buff[1] + data["repo"] = buff[2] + tree = GetItemFromList(buff, 3) + if tree: + data["branch"] = GetItemFromList(buff, 4) + else: + data["branch"] = arg1 + if arg2: + data["branch"] = arg2 + return data +#end define + +def check_vport(local, ton): vconfig = ton.GetValidatorConfig() addr = vconfig.addrs.pop() ip = int2ip(addr.ip) @@ -133,26 +235,26 @@ def check_git(input_args, default_repo, text): git_path = f"{src_dir}/{default_repo}" default_author = "ton-blockchain" default_branch = "master" - + # Get author, repo, branch local_author, local_repo = get_git_author_and_repo(git_path) local_branch = get_git_branch(git_path) - + # Set author, repo, branch data = GetAuthorRepoBranchFromArgs(input_args) need_author = data.get("author") need_repo = data.get("repo") need_branch = data.get("branch") - + # Check if remote repo is different from default if ((need_author is None and local_author != default_author) or (need_repo is None and local_repo != default_repo)): remote_url = f"https://github.com/{local_author}/{local_repo}/tree/{need_branch if need_branch else local_branch}" - raise Exception(f"{text} error: You are on {remote_url} remote url, to {text} to the tip use `{text} {remote_url}` command") + raise Exception(f"{text} error: You are on {remote_url} remote url, to update to the tip use `{text} {remote_url}` command") elif need_branch is None and local_branch != default_branch: - raise Exception(f"{text} error: You are on {local_branch} branch, to {text} to the tip of {local_branch} branch use `{text} {local_branch}` command") + raise Exception(f"{text} error: You are on {local_branch} branch, to update to the tip of {local_branch} branch use `{text} {local_branch}` command") #end if - + if need_author is None: need_author = local_author if need_repo is None: @@ -160,36 +262,17 @@ def check_git(input_args, default_repo, text): if need_branch is None: need_branch = local_branch #end if - - return need_author, need_repo, need_branch -#end define -def GetAuthorRepoBranchFromArgs(args): - data = dict() - arg1 = GetItemFromList(args, 0) - arg2 = GetItemFromList(args, 1) - if arg1: - if "https://" in arg1: - buff = arg1[8:].split('/') - print(f"buff: {buff}") - data["author"] = buff[1] - data["repo"] = buff[2] - tree = GetItemFromList(buff, 3) - if tree: - data["branch"] = GetItemFromList(buff, 4) - else: - data["branch"] = arg1 - if arg2: - data["branch"] = arg2 - return data + return need_author, need_repo, need_branch #end define -def Update(args): +def Update(local, args): repo = "mytonctrl" author, repo, branch = check_git(args, repo, "update") # Run script - runArgs = ["bash", "/usr/src/mytonctrl/scripts/update.sh", "-a", author, "-r", repo, "-b", branch] + update_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/update.sh') + runArgs = ["bash", update_script_path, "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) if exitCode == 0: text = "Update - {green}OK{endc}" @@ -199,10 +282,10 @@ def Update(args): local.exit() #end define -def Upgrade(args): +def Upgrade(ton, args): repo = "ton" author, repo, branch = check_git(args, repo, "upgrade") - + # bugfix if the files are in the wrong place liteClient = ton.GetSettings("liteClient") configPath = liteClient.get("configPath") @@ -220,11 +303,11 @@ def Upgrade(args): if "/usr/bin/ton" in pubKeyPath: validatorConsole["pubKeyPath"] = "/var/ton-work/keys/server.pub" ton.SetSettings("validatorConsole", validatorConsole) - + # Run script - runArgs = ["bash", "/usr/src/mytonctrl/scripts/upgrade.sh", "-a", author, "-r", repo, "-b", branch] + upgrade_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/upgrade.sh') + runArgs = ["bash", upgrade_script_path, "-a", author, "-r", repo, "-b", branch] exitCode = run_as_root(runArgs) - exitCode += run_as_root(["python3", "/usr/src/mytonctrl/scripts/upgrade.py"]) if exitCode == 0: text = "Upgrade - {green}OK{endc}" else: @@ -232,21 +315,29 @@ def Upgrade(args): color_print(text) #end define -def CheckMytonctrlUpdate(): +def CheckMytonctrlUpdate(local): git_path = local.buffer.my_dir result = check_git_update(git_path) if result is True: color_print(local.translate("mytonctrl_update_available")) #end define -def CheckTonUpdate(): +def CheckTonUpdate(local): git_path = "/usr/src/ton" result = check_git_update(git_path) if result is True: color_print(local.translate("ton_update_available")) #end define -def PrintStatus(args): +def PrintTest(local, args): + print(json.dumps(local.buffer, indent=2)) +#end define + +def sl(ton, args): + Slashing(ton.local, ton) +#end define + +def PrintStatus(local, ton, args): opt = None if len(args) == 1: opt = args[0] @@ -289,13 +380,13 @@ def PrintStatus(args): validatorAccount = ton.GetAccount(validatorWallet.addrB64) else: validatorAccount = None - PrintTonStatus(startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg) - PrintLocalStatus(adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validatorStatus, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg) - PrintTonConfig(fullConfigAddr, fullElectorAddr, config15, config17) - PrintTimes(rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, config15) + PrintTonStatus(local, startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg) + PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validatorStatus, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg) + PrintTonConfig(local, fullConfigAddr, fullElectorAddr, config15, config17) + PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, config15) #end define -def PrintTonStatus(startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg): +def PrintTonStatus(local, startWorkTime, totalValidators, onlineValidators, shardsNumber, offersNumber, complaintsNumber, tpsAvg): tps1 = tpsAvg[0] tps5 = tpsAvg[1] tps15 = tpsAvg[2] @@ -335,7 +426,7 @@ def PrintTonStatus(startWorkTime, totalValidators, onlineValidators, shardsNumbe print() #end define -def PrintLocalStatus(adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validatorStatus, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg): +def PrintLocalStatus(local, adnlAddr, validatorIndex, validatorEfficiency, validatorWallet, validatorAccount, validatorStatus, dbSize, dbUsage, memoryInfo, swapInfo, netLoadAvg, disksLoadAvg, disksLoadPercentAvg): if validatorWallet is None: return walletAddr = validatorWallet.addrB64 @@ -389,11 +480,11 @@ def PrintLocalStatus(adnlAddr, validatorIndex, validatorEfficiency, validatorWal # Disks status disksLoad_data = list() for key, item in disksLoadAvg.items(): - diskLoad1_text = bcolors.green_text(item[0]) - diskLoad5_text = bcolors.green_text(item[1]) + diskLoad1_text = bcolors.green_text(item[0]) # TODO: this variables is unused. Why? + diskLoad5_text = bcolors.green_text(item[1]) # TODO: this variables is unused. Why? diskLoad15_text = bcolors.green_text(item[2]) - diskLoadPercent1_text = GetColorInt(disksLoadPercentAvg[key][0], 80, logic="less", ending="%") - diskLoadPercent5_text = GetColorInt(disksLoadPercentAvg[key][1], 80, logic="less", ending="%") + diskLoadPercent1_text = GetColorInt(disksLoadPercentAvg[key][0], 80, logic="less", ending="%") # TODO: this variables is unused. Why? + diskLoadPercent5_text = GetColorInt(disksLoadPercentAvg[key][1], 80, logic="less", ending="%") # TODO: this variables is unused. Why? diskLoadPercent15_text = GetColorInt(disksLoadPercentAvg[key][2], 80, logic="less", ending="%") buff = "{}, {}" buff = "{}{}:[{}{}{}]{}".format(bcolors.cyan, key, bcolors.default, buff, bcolors.cyan, bcolors.endc) @@ -477,7 +568,7 @@ def GetColorStatus(input): return result #end define -def PrintTonConfig(fullConfigAddr, fullElectorAddr, config15, config17): +def PrintTonConfig(local, fullConfigAddr, fullElectorAddr, config15, config17): validatorsElectedFor = config15["validatorsElectedFor"] electionsStartBefore = config15["electionsStartBefore"] electionsEndBefore = config15["electionsEndBefore"] @@ -504,7 +595,7 @@ def PrintTonConfig(fullConfigAddr, fullElectorAddr, config15, config17): print() #end define -def PrintTimes(rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, config15): +def PrintTimes(local, rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, config15): validatorsElectedFor = config15["validatorsElectedFor"] electionsStartBefore = config15["electionsStartBefore"] electionsEndBefore = config15["electionsEndBefore"] @@ -546,7 +637,7 @@ def PrintTimes(rootWorkchainEnabledTime_int, startWorkTime, oldStartWorkTime, co #end define def GetColorTime(datetime, timestamp): - newTimestamp = get_timestamp() + newTimestamp = GetTimestamp() if timestamp > newTimestamp: result = bcolors.green_text(datetime) else: @@ -554,7 +645,7 @@ def GetColorTime(datetime, timestamp): return result #end define -def Seqno(args): +def Seqno(ton, args): try: walletName = args[0] except: @@ -565,7 +656,7 @@ def Seqno(args): print(walletName, "seqno:", seqno) #end define -def CreatNewWallet(args): +def CreatNewWallet(ton, args): version = "v1" try: if len(args) == 0: @@ -587,10 +678,10 @@ def CreatNewWallet(args): table = list() table += [["Name", "Workchain", "Address"]] table += [[wallet.name, wallet.workchain, wallet.addrB64_init]] - print_table(table) + PrintTable(table) #end define -def ActivateWallet(args): +def ActivateWallet(local, ton, args): try: walletName = args[0] except Exception as err: @@ -606,7 +697,7 @@ def ActivateWallet(args): color_print("ActivateWallet - {green}OK{endc}") #end define -def PrintWalletsList(args): +def PrintWalletsList(ton, args): table = list() table += [["Name", "Status", "Balance", "Ver", "Wch", "Address"]] data = ton.GetWallets() @@ -618,10 +709,10 @@ def PrintWalletsList(args): if account.status != "active": wallet.addrB64 = wallet.addrB64_init table += [[wallet.name, account.status, account.balance, wallet.version, wallet.workchain, wallet.addrB64]] - print_table(table) + PrintTable(table) #end define -def ImportWalletFromFile(args): +def ImportWalletFromFile(local, ton, args): try: filePath = args[0] except: @@ -646,7 +737,7 @@ def ImportWalletFromFile(args): color_print("ImportWalletFromFile - {green}OK{endc}") #end define -def ImportWallet(args): +def ImportWallet(ton, args): try: addr = args[0] key = args[1] @@ -657,7 +748,7 @@ def ImportWallet(args): print("Wallet name:", name) #end define -def SetWalletVersion(args): +def SetWalletVersion(ton, args): try: addr = args[0] version = args[1] @@ -668,7 +759,7 @@ def SetWalletVersion(args): color_print("SetWalletVersion - {green}OK{endc}") #end define -def ExportWallet(args): +def ExportWallet(ton, args): try: name = args[0] except: @@ -680,7 +771,7 @@ def ExportWallet(args): print("Secret key:", key) #end define -def DeleteWallet(args): +def DeleteWallet(ton, args): try: walletName = args[0] except: @@ -694,7 +785,7 @@ def DeleteWallet(args): color_print("DeleteWallet - {green}OK{endc}") #end define -def ViewAccountStatus(args): +def ViewAccountStatus(ton, args): try: addrB64 = args[0] except: @@ -706,24 +797,24 @@ def ViewAccountStatus(args): statusTable = list() statusTable += [["Address", "Status", "Version", "Balance"]] statusTable += [[addrB64, account.status, version, account.balance]] - historyTable = GetHistoryTable(addrB64, 10) - print_table(statusTable) + historyTable = GetHistoryTable(ton, addrB64, 10) + PrintTable(statusTable) print() - print_table(historyTable) + PrintTable(historyTable) #end define -def ViewAccountHistory(args): +def ViewAccountHistory(ton, args): try: addr = args[0] limit = int(args[1]) except: color_print("{red}Bad args. Usage:{endc} vah ") return - table = GetHistoryTable(addr, limit) - print_table(table) + table = GetHistoryTable(ton, addr, limit) + PrintTable(table) #end define -def GetHistoryTable(addr, limit): +def GetHistoryTable(ton, addr, limit): addr = ton.GetDestinationAddr(addr) account = ton.GetAccount(addr) history = ton.GetAccountHistory(account, limit) @@ -748,7 +839,7 @@ def GetHistoryTable(addr, limit): return table #end define -def MoveCoins(args): +def MoveCoins(ton, args): try: walletName = args[0] destination = args[1] @@ -763,7 +854,7 @@ def MoveCoins(args): color_print("MoveCoins - {green}OK{endc}") #end define -def MoveCoinsThroughProxy(args): +def MoveCoinsThroughProxy(ton, args): try: walletName = args[0] destination = args[1] @@ -777,7 +868,7 @@ def MoveCoinsThroughProxy(args): color_print("MoveCoinsThroughProxy - {green}OK{endc}") #end define -def CreatNewBookmark(args): +def CreatNewBookmark(ton, args): try: name = args[0] addr = args[1] @@ -798,7 +889,7 @@ def CreatNewBookmark(args): color_print("CreatNewBookmark - {green}OK{endc}") #end define -def PrintBookmarksList(args): +def PrintBookmarksList(ton, args): data = ton.GetBookmarks() if (data is None or len(data) == 0): print("No data") @@ -811,10 +902,10 @@ def PrintBookmarksList(args): addr = item.get("addr") data = item.get("data") table += [[name, type, addr, data]] - print_table(table) + PrintTable(table) #end define -def DeleteBookmark(args): +def DeleteBookmark(ton, args): try: name = args[0] type = args[1] @@ -825,7 +916,37 @@ def DeleteBookmark(args): color_print("DeleteBookmark - {green}OK{endc}") #end define -def PrintOffersList(args): +# def CreatNewAutoTransferRule(args): +# try: +# name = args[0] +# addr = args[1] +# except: +# color_print("{red}Bad args. Usage:{endc} nr ") +# return +# rule = dict() +# rule["name"] = name +# rule["addr"] = addr +# ton.AddAutoTransferRule(rule) +# color_print("CreatNewAutoTransferRule - {green}OK{endc}") +# #end define + +# def PrintAutoTransferRulesList(args): +# data = ton.GetRules() +# if (data is None or len(data) == 0): +# print("No data") +# return +# table = list() +# table += [["Name", "fix me"]] +# for item in data: +# table += [[item.get("name"), item.get("fix me")]] +# PrintTable(table) +# #end define + +# def DeleteAutoTransferRule(args): +# print("fix me") +# #end define + +def PrintOffersList(ton, args): offers = ton.GetOffers() if "--json" in args: text = json.dumps(offers, indent=2) @@ -849,10 +970,10 @@ def PrintOffersList(args): if isPassed == False: isPassed = bcolors.red_text("false") table += [[hash, votedValidators, wl, approvedPercent_text, isPassed]] - print_table(table) + PrintTable(table) #end define -def VoteOffer(args): +def VoteOffer(ton, args): if len(args) == 0: color_print("{red}Bad args. Usage:{endc} vo ") return @@ -861,7 +982,7 @@ def VoteOffer(args): color_print("VoteOffer - {green}OK{endc}") #end define -def OfferDiff(args): +def OfferDiff(ton, args): try: offerHash = args[0] offerHash = offerHash @@ -871,7 +992,7 @@ def OfferDiff(args): ton.GetOfferDiff(offerHash) #end define -def GetConfig(args): +def GetConfig(ton, args): try: configId = args[0] configId = int(configId) @@ -883,7 +1004,7 @@ def GetConfig(args): print(text) #end define -def PrintComplaintsList(args): +def PrintComplaintsList(ton, args): past = "past" in args complaints = ton.GetComplaints(past=past) if "--json" in args: @@ -909,10 +1030,10 @@ def PrintComplaintsList(args): if isPassed == False: isPassed = bcolors.red_text("false") table += [[electionId, adnl, Fine_text, votedValidators, approvedPercent_text, isPassed]] - print_table(table) + PrintTable(table) #end define -def VoteComplaint(args): +def VoteComplaint(ton, args): try: electionId = args[0] complaintHash = args[1] @@ -923,7 +1044,7 @@ def VoteComplaint(args): color_print("VoteComplaint - {green}OK{endc}") #end define -def NewDomain(args): +def NewDomain(ton, args): try: domainName = args[0] walletName = args[1] @@ -939,7 +1060,7 @@ def NewDomain(args): color_print("NewDomain - {green}OK{endc}") #end define -def PrintDomainsList(args): +def PrintDomainsList(ton, args): data = ton.GetDomains() if (data is None or len(data) == 0): print("No data") @@ -953,10 +1074,10 @@ def PrintDomainsList(args): endTime = timestamp2datetime(endTime, "%d.%m.%Y") adnlAddr = item.get("adnlAddr") table += [[domainName, walletName, endTime, adnlAddr]] - print_table(table) + PrintTable(table) #end define -def ViewDomainStatus(args): +def ViewDomainStatus(ton, args): try: domainName = args[0] except: @@ -972,7 +1093,7 @@ def ViewDomainStatus(args): print_table(table) #end define -def DeleteDomain(args): +def DeleteDomain(ton, args): try: domainName = args[0] except: @@ -982,7 +1103,7 @@ def DeleteDomain(args): color_print("DeleteDomain - {green}OK{endc}") #end define -def GetDomainFromAuction(args): +def GetDomainFromAuction(ton, args): try: walletName = args[0] addr = args[1] @@ -993,7 +1114,7 @@ def GetDomainFromAuction(args): color_print("GetDomainFromAuction - {green}OK{endc}") #end define -def PrintElectionEntriesList(args): +def PrintElectionEntriesList(ton, args): past = "past" in args entries = ton.GetElectionEntries(past=past) if "--json" in args: @@ -1018,12 +1139,12 @@ def PrintElectionEntriesList(args): print_table(table) #end define -def VoteElectionEntry(args): - Elections(ton) +def VoteElectionEntry(ton, args): + Elections(ton.local, ton) color_print("VoteElectionEntry - {green}OK{endc}") #end define -def PrintValidatorList(args): +def PrintValidatorList(ton, args): past = "past" in args validators = ton.GetValidatorsList(past=past) if "--json" in args: @@ -1064,7 +1185,7 @@ def Reduct(item): return result #end define -def GetSettings(args): +def GetSettings(ton, args): try: name = args[0] except: @@ -1074,14 +1195,14 @@ def GetSettings(args): print(json.dumps(result, indent=2)) #end define -def SetSettings(args): +def SetSettings(ton, args): try: name = args[0] value = args[1] except: color_print("{red}Bad args. Usage:{endc} set ") return - result = ton.SetSettings(name, value) + ton.SetSettings(name, value) color_print("SetSettings - {green}OK{endc}") #end define @@ -1089,7 +1210,8 @@ def Xrestart(inputArgs): if len(inputArgs) < 2: color_print("{red}Bad args. Usage:{endc} xrestart ") return - args = ["python3", "/usr/src/mytonctrl/scripts/xrestart.py"] + xrestart_script_path = pkg_resources.resource_filename('mytonctrl', 'scripts/xrestart.py') + args = ["python3", xrestart_script_path] # TODO: Fix path args += inputArgs exitCode = run_as_root(args) if exitCode == 0: @@ -1103,7 +1225,102 @@ def Xlist(args): color_print("Xlist - {green}OK{endc}") #end define -def NewPool(args): +def GetPubKey(ton, args): + adnlAddr = ton.GetAdnlAddr() + pubkey = ton.GetPubKey(adnlAddr) + print("pubkey:", pubkey) +#end define + +def SignShardOverlayCert(ton, args): + try: + adnl = args[0] + pubkey = args[0] + except: + color_print("{red}Bad args. Usage:{endc} ssoc ") + return + ton.SignShardOverlayCert(adnl, pubkey) +#end define + +def ImportShardOverlayCert(ton, args): + ton.ImportShardOverlayCert() +#end define + +def NewNominationController(ton, args): + try: + name = args[0] + nominatorAddr = args[1] + rewardShare = args[2] + coverAbility = args[3] + except: + color_print("{red}Bad args. Usage:{endc} new_controller ") + return + ton.CreateNominationController(name, nominatorAddr, rewardShare=rewardShare, coverAbility=coverAbility) + color_print("NewNominationController - {green}OK{endc}") +#end define + +def GetNominationControllerData(ton, args): + try: + addrB64 = args[0] + except: + color_print("{red}Bad args. Usage:{endc} get_nomination_controller_data ") + return + addrB64 = ton.GetDestinationAddr(addrB64) + controllerData = ton.GetControllerData(addrB64) + print(json.dumps(controllerData, indent=4)) +#end define + +def DepositToNominationController(ton, args): + try: + walletName = args[0] + destination = args[1] + amount = float(args[2]) + except: + color_print("{red}Bad args. Usage:{endc} add_to_nomination_controller ") + return + destination = ton.GetDestinationAddr(destination) + ton.DepositToNominationController(walletName, destination, amount) + color_print("DepositToNominationController - {green}OK{endc}") +#end define + +def WithdrawFromNominationController(ton, args): + try: + walletName = args[0] + destination = args[1] + amount = float(args[2]) + except: + color_print("{red}Bad args. Usage:{endc} withdraw_from_nomination_controller ") + return + destination = ton.GetDestinationAddr(destination) + ton.WithdrawFromNominationController(walletName, destination, amount) + color_print("WithdrawFromNominationController - {green}OK{endc}") +#end define + +def SendRequestToNominationController(ton, args): + try: + walletName = args[0] + destination = args[1] + except: + color_print("{red}Bad args. Usage:{endc} request_to_nomination_controller ") + return + destination = ton.GetDestinationAddr(destination) + ton.SendRequestToNominationController(walletName, destination) + color_print("SendRequestToNominationController - {green}OK{endc}") +#end define + +def NewRestrictedWallet(ton, args): + try: + workchain = int(args[0]) + name = args[1] + ownerAddr = args[2] + #subwallet = args[3] + except: + color_print("{red}Bad args. Usage:{endc} new_restricted_wallet ") + return + ton.CreateRestrictedWallet(name, ownerAddr, workchain=workchain) + color_print("NewRestrictedWallet - {green}OK{endc}") +#end define + +def NewPool(ton, args): try: poolName = args[0] validatorRewardSharePercent = float(args[1]) @@ -1117,7 +1334,7 @@ def NewPool(args): color_print("NewPool - {green}OK{endc}") #end define -def ActivatePool(args): +def ActivatePool(local, ton, args): try: poolName = args[0] except: @@ -1131,7 +1348,7 @@ def ActivatePool(args): color_print("ActivatePool - {green}OK{endc}") #end define -def PrintPoolsList(args): +def PrintPoolsList(ton, args): table = list() table += [["Name", "Status", "Balance", "Address"]] data = ton.GetPools() @@ -1146,7 +1363,7 @@ def PrintPoolsList(args): print_table(table) #end define -def GetPoolData(args): +def GetPoolData(ton, args): try: poolName = args[0] except: @@ -1161,7 +1378,7 @@ def GetPoolData(args): print(json.dumps(poolData, indent=4)) #end define -def DepositToPool(args): +def DepositToPool(ton, args): try: walletName = args[0] pollAddr = args[1] @@ -1173,7 +1390,7 @@ def DepositToPool(args): color_print("DepositToPool - {green}OK{endc}") #end define -def WithdrawFromPool(args): +def WithdrawFromPool(ton, args): try: poolAddr = args[0] amount = float(args[1]) @@ -1184,7 +1401,7 @@ def WithdrawFromPool(args): color_print("WithdrawFromPool - {green}OK{endc}") #end define -def DeletePool(args): +def DeletePool(ton, args): try: poolName = args[0] except: @@ -1195,14 +1412,14 @@ def DeletePool(args): color_print("DeletePool - {green}OK{endc}") #end define -def UpdateValidatorSet(args): +def UpdateValidatorSet(ton, args): try: poolAddr = args[0] except: color_print("{red}Bad args. Usage:{endc} update_validator_set ") return - wallet = self.GetValidatorWallet() - self.PoolUpdateValidatorSet(poolAddr, wallet) + wallet = ton.GetValidatorWallet() + ton.PoolUpdateValidatorSet(poolAddr, wallet) color_print("DeletePool - {green}OK{endc}") #end define @@ -1211,7 +1428,12 @@ def UpdateValidatorSet(args): ### Start of the program ### -if __name__ == "__main__": - Init(sys.argv[1:]) +def mytonctrl(): + local = MyPyClass('mytonctrl.py') + mytoncore_local = MyPyClass('mytoncore.py') + ton = MyTonCore(mytoncore_local) + console = MyPyConsole() + + Init(local, ton, console, sys.argv[1:]) console.Run() -#end if +#end define diff --git a/translate.json b/mytonctrl/resources/translate.json similarity index 100% rename from translate.json rename to mytonctrl/resources/translate.json diff --git a/scripts/update.sh b/mytonctrl/scripts/update.sh similarity index 73% rename from scripts/update.sh rename to mytonctrl/scripts/update.sh index 6c17a217..e6b3f680 100644 --- a/scripts/update.sh +++ b/mytonctrl/scripts/update.sh @@ -28,17 +28,23 @@ done COLOR='\033[92m' ENDC='\033[0m' -# Установка компонентов python3 -pip3 install fastcrc - # Go to work dir cd ${srcdir} rm -rf ${srcdir}/${repo} # Update code echo "https://github.com/${author}/${repo}.git -> ${branch}" -git clone --recursive https://github.com/${author}/${repo}.git -cd ${repo} && git checkout ${branch} && git submodule update --init --recursive +git clone https://github.com/${author}/${repo}.git +cd ${repo} +git checkout ${branch} +git submodule update --init --recursive + +# FIXME: add __init__.py in these repos +touch mypyconsole/__init__.py +touch mypylib/__init__.py + +pip3 install -U . + systemctl restart mytoncore # Конец diff --git a/scripts/upgrade.sh b/mytonctrl/scripts/upgrade.sh similarity index 92% rename from scripts/upgrade.sh rename to mytonctrl/scripts/upgrade.sh index 03ac0b44..2e845b03 100644 --- a/scripts/upgrade.sh +++ b/mytonctrl/scripts/upgrade.sh @@ -42,6 +42,10 @@ else cp /usr/bin/ton/validator-engine-console/client.pub /var/ton-work/keys/client.pub cp /usr/bin/ton/validator-engine-console/server.pub /var/ton-work/keys/server.pub cp /usr/bin/ton/validator-engine-console/liteserver.pub /var/ton-work/keys/liteserver.pub + + # fix validator.service + sed -i 's/validator-engine\/ton-global.config.json/global.config.json/' /etc/systemd/system/validator.service + systemctl daemon-reload fi # Go to work dir diff --git a/scripts/xrestart.py b/mytonctrl/scripts/xrestart.py similarity index 99% rename from scripts/xrestart.py rename to mytonctrl/scripts/xrestart.py index b367eb1f..797835a6 100644 --- a/scripts/xrestart.py +++ b/mytonctrl/scripts/xrestart.py @@ -70,4 +70,4 @@ def result(): if __name__ == "__main__": Xguard() -#end if \ No newline at end of file +#end if diff --git a/mytoninstaller/__init__.py b/mytoninstaller/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mytoninstaller/__main__.py b/mytoninstaller/__main__.py new file mode 100644 index 00000000..78efe3f2 --- /dev/null +++ b/mytoninstaller/__main__.py @@ -0,0 +1,5 @@ +from mytoninstaller.mytoninstaller import mytoninstaller + + +if __name__ == '__main__': + mytoninstaller() diff --git a/mytoninstaller/config.py b/mytoninstaller/config.py new file mode 100644 index 00000000..c1a78dd2 --- /dev/null +++ b/mytoninstaller/config.py @@ -0,0 +1,136 @@ +import os +import json +import subprocess +import requests +import base64 + +from mytoncore.utils import hex2b64, dict2b64 +from mytoninstaller.utils import StartMytoncore, GetInitBlock +from mypylib.mypylib import ip2int, Dict + + +defaultLocalConfigPath = "/usr/bin/ton/local.config.json" + + +def GetConfig(**kwargs): + path = kwargs.get("path") + file = open(path, 'rt') + text = file.read() + file.close() + config = Dict(json.loads(text)) + return config +#end define + + +def SetConfig(**kwargs): + path = kwargs.get("path") + data = kwargs.get("data") + + # write config + text = json.dumps(data, indent=4) + file = open(path, 'wt') + file.write(text) + file.close() +#end define + + +def BackupVconfig(local): + local.add_log("Backup validator config file 'config.json' to 'config.json.backup'", "debug") + vconfig_path = local.buffer.vconfig_path + backupPath = vconfig_path + ".backup" + args = ["cp", vconfig_path, backupPath] + subprocess.run(args) +#end define + + +def BackupMconfig(local): + local.add_log("Backup mytoncore config file 'mytoncore.db' to 'mytoncore.db.backup'", "debug") + mconfig_path = local.buffer.mconfig_path + backupPath = mconfig_path + ".backup" + args = ["cp", mconfig_path, backupPath] + subprocess.run(args) +#end define + + +def GetPortsFromVconfig(local): + vconfig_path = local.buffer.vconfig_path + + # read vconfig + local.add_log("read vconfig", "debug") + vconfig = GetConfig(path=vconfig_path) + + # read mconfig + local.add_log("read mconfig", "debug") + mconfig_path = local.buffer.mconfig_path + mconfig = GetConfig(path=mconfig_path) + + # edit mytoncore config file + local.add_log("edit mytoncore config file", "debug") + mconfig.liteClient.liteServer.port = mconfig.liteservers[0].port + mconfig.validatorConsole.addr = f"127.0.0.1:{mconfig.control[0].port}" + + # write mconfig + local.add_log("write mconfig", "debug") + SetConfig(path=mconfig_path, data=mconfig) + + # restart mytoncore + StartMytoncore(local) +#end define + + +def CreateLocalConfig(local, initBlock, localConfigPath=defaultLocalConfigPath): + # dirty hack, but GetInitBlock() function uses the same technique + from mytoncore import hex2base64 + + # read global config file + file = open("/usr/bin/ton/global.config.json", 'rt') + text = file.read() + data = json.loads(text) + file.close() + + # edit config + liteServerConfig = GetLiteServerConfig(local) + data["liteservers"] = [liteServerConfig] + data["validator"]["init_block"]["seqno"] = initBlock["seqno"] + data["validator"]["init_block"]["root_hash"] = hex2base64(initBlock["rootHash"]) + data["validator"]["init_block"]["file_hash"] = hex2base64(initBlock["fileHash"]) + text = json.dumps(data, indent=4) + + # write local config file + file = open(localConfigPath, 'wt') + file.write(text) + file.close() + + # chown + user = local.buffer.user + args = ["chown", "-R", user + ':' + user, localConfigPath] + + print("Local config file created:", localConfigPath) +#end define + + +def get_own_ip(): + requests.packages.urllib3.util.connection.HAS_IPV6 = False + ip = requests.get("https://ifconfig.me/ip").text + return ip +#end define + + +def GetLiteServerConfig(local): + keys_dir = local.buffer.keys_dir + liteserver_key = keys_dir + "liteserver" + liteserver_pubkey = liteserver_key + ".pub" + result = Dict() + file = open(liteserver_pubkey, 'rb') + data = file.read() + file.close() + key = base64.b64encode(data[4:]) + ip = get_own_ip() + mconfig = GetConfig(path=local.buffer.mconfig_path) + result.ip = ip2int(ip) + result.port = mconfig.liteClient.liteServer.port + result.id = Dict() + result.id["@type"]= "pub.ed25519" + result.id.key= key.decode() + return result +#end define diff --git a/mytoninstaller/mytoninstaller.py b/mytoninstaller/mytoninstaller.py new file mode 100644 index 00000000..f284e61e --- /dev/null +++ b/mytoninstaller/mytoninstaller.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# -*- coding: utf_8 -*- + +import os, sys +import inspect +import random +import json +import subprocess + +from mypylib.mypylib import MyPyClass, run_as_root +from mypyconsole.mypyconsole import MyPyConsole + +from mytoninstaller.config import GetLiteServerConfig +from mytoninstaller.utils import GetInitBlock +from mytoncore.utils import dict2b64, str2bool, b642dict + +from mytoninstaller.settings import ( + FirstNodeSettings, + FirstMytoncoreSettings, + EnableValidatorConsole, + EnableLiteServer, + EnableDhtServer, + EnableJsonRpc, + EnablePytonv3, + EnableTonHttpApi, + DangerousRecoveryValidatorConfigFile, + CreateSymlinks, +) +from mytoninstaller.config import ( + CreateLocalConfig, + BackupVconfig, + BackupMconfig, +) + +from functools import partial + + +def Init(local, console): + local.db.config.isStartOnlyOneProcess = False + local.db.config.logLevel = "debug" + local.db.config.isIgnorLogWarning = True # disable warning + local.run() + local.db.config.isIgnorLogWarning = False # enable warning + + + # create variables + user = os.environ.get("USER", "root") + local.buffer.user = user + local.buffer.vuser = "validator" + local.buffer.cport = random.randint(2000, 65000) + local.buffer.lport = random.randint(2000, 65000) + + # this funciton injects MyPyClass instance + def inject_globals(func): + args = [] + for arg_name in inspect.getfullargspec(func)[0]: + if arg_name == 'local': + args.append(local) + return partial(func, *args) + + # Create user console + console.name = "MyTonInstaller" + console.color = console.RED + console.AddItem("status", inject_globals(Status), "Print TON component status") + console.AddItem("enable", inject_globals(Enable), "Enable some function: 'FN' - Full node, 'VC' - Validator console, 'LS' - Liteserver, 'DS' - DHT-Server, 'JR' - jsonrpc, 'PT' - pyTONv3. Example: 'enable FN'") + console.AddItem("update", inject_globals(Enable), "Update some function: 'JR' - jsonrpc. Example: 'update JR'") + console.AddItem("plsc", inject_globals(PrintLiteServerConfig), "Print LiteServer config") + console.AddItem("clcf", inject_globals(CreateLocalConfigFile), "CreateLocalConfigFile") + console.AddItem("drvcf", inject_globals(DRVCF), "Dangerous recovery validator config file") + console.AddItem("setwebpass", inject_globals(SetWebPassword), "Set a password for the web admin interface") + + Refresh(local) +#end define + + +def Refresh(local): + user = local.buffer.user + local.buffer.mconfig_path = "/home/{user}/.local/share/mytoncore/mytoncore.db".format(user=user) + if user == 'root': + local.buffer.mconfig_path = "/usr/local/bin/mytoncore/mytoncore.db" + #end if + + # create variables + bin_dir = "/usr/bin/" + src_dir = "/usr/src/" + ton_work_dir = "/var/ton-work/" + ton_bin_dir = bin_dir + "ton/" + ton_src_dir = src_dir + "ton/" + local.buffer.bin_dir = bin_dir + local.buffer.src_dir = src_dir + local.buffer.ton_work_dir = ton_work_dir + local.buffer.ton_bin_dir = ton_bin_dir + local.buffer.ton_src_dir = ton_src_dir + ton_db_dir = ton_work_dir + "db/" + keys_dir = ton_work_dir + "keys/" + local.buffer.ton_db_dir = ton_db_dir + local.buffer.keys_dir = keys_dir + local.buffer.ton_log_path = ton_work_dir + "log" + local.buffer.validator_app_path = ton_bin_dir + "validator-engine/validator-engine" + local.buffer.global_config_path = ton_bin_dir + "global.config.json" + local.buffer.vconfig_path = ton_db_dir + "config.json" +#end define + + +def Status(local, args): + keys_dir = local.buffer.keys_dir + server_key = keys_dir + "server" + client_key = keys_dir + "client" + liteserver_key = keys_dir + "liteserver" + liteserver_pubkey = liteserver_key + ".pub" + + + fnStatus = os.path.isfile(local.buffer.vconfig_path) + mtcStatus = os.path.isfile(local.buffer.mconfig_path) + vcStatus = os.path.isfile(server_key) or os.path.isfile(client_key) + lsStatus = os.path.isfile(liteserver_pubkey) + + print("Full node status:", fnStatus) + print("Mytoncore status:", mtcStatus) + print("V.console status:", vcStatus) + print("Liteserver status:", lsStatus) +#end define + + +def Enable(local, args): + name = args[0] + if name == "PT": + CreateLocalConfigFile(local, args) + args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "enable{name}".format(name=name)] + run_as_root(args) +#end define + + +def DRVCF(local, args): + user = local.buffer["user"] + args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "drvcf"] + run_as_root(args) +#end define + + +def SetWebPassword(args): + args = ["python3", "/usr/src/mtc-jsonrpc/mtc-jsonrpc.py", "-p"] + subprocess.run(args) +#end define + + +def PrintLiteServerConfig(local, args): + liteServerConfig = GetLiteServerConfig(local) + text = json.dumps(liteServerConfig, indent=4) + print(text) +#end define + + +def CreateLocalConfigFile(local, args): + initBlock = GetInitBlock() + initBlock_b64 = dict2b64(initBlock) + user = local.buffer["user"] + args = ["python3", "-m", "mytoninstaller", "-u", local.buffer.user, "-e", "clc", "-i", initBlock_b64] + run_as_root(args) +#end define + + +def Event(local, name): + if name == "enableFN": + FirstNodeSettings(local) + if name == "enableVC": + EnableValidatorConsole(local) + if name == "enableLS": + EnableLiteServer(local) + if name == "enableDS": + EnableDhtServer(local) + if name == "drvcf": + DangerousRecoveryValidatorConfigFile(local) + if name == "enableJR": + EnableJsonRpc(local) + if name == "enablePT": + # EnablePytonv3(local) + EnableTonHttpApi(local) + if name == "clc": + ix = sys.argv.index("-i") + initBlock_b64 = sys.argv[ix+1] + initBlock = b642dict(initBlock_b64) + CreateLocalConfig(local, initBlock) +#end define + + +def General(local): + if "-u" in sys.argv: + ux = sys.argv.index("-u") + user = sys.argv[ux+1] + local.buffer.user = user + Refresh(local) + if "-e" in sys.argv: + ex = sys.argv.index("-e") + name = sys.argv[ex+1] + Event(local, name) + if "-m" in sys.argv: + mx = sys.argv.index("-m") + mode = sys.argv[mx+1] + if "-t" in sys.argv: + mx = sys.argv.index("-t") + telemetry = sys.argv[mx+1] + local.buffer.telemetry = str2bool(telemetry) + if "--dump" in sys.argv: + mx = sys.argv.index("--dump") + dump = sys.argv[mx+1] + local.buffer.dump = str2bool(dump) + #end if + + # Создать настройки для mytoncore.py + FirstMytoncoreSettings(local) + + if mode == "full": + FirstNodeSettings(local) + EnableValidatorConsole(local) + EnableLiteServer(local) + BackupVconfig(local) + BackupMconfig(local) + #end if + + # Создать символические ссылки + CreateSymlinks(local) + #end if +#end define + + +### +### Start of the program +### +def mytoninstaller(): + local = MyPyClass(__file__) + console = MyPyConsole() + + Init(local, console) + if len(sys.argv) > 1: + General(local) + else: + console.Run() + local.exit() diff --git a/mytoninstaller/scripts/__init__.py b/mytoninstaller/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mytoninstaller/scripts/add2systemd.sh b/mytoninstaller/scripts/add2systemd.sh new file mode 100755 index 00000000..13626522 --- /dev/null +++ b/mytoninstaller/scripts/add2systemd.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +# Проверить sudo +if [ "$(id -u)" != "0" ]; then + echo "Please run script as root" + exit 1 +fi + +post="/bin/echo service down" +user=root +group=root + +while getopts n:s:p:u:g: flag +do + case "${flag}" in + n) name=${OPTARG};; + s) start=${OPTARG};; + p) post=${OPTARG};; + u) user=${OPTARG};; + g) group=${OPTARG};; + esac +done + +if [ -z "$name" ]; then + echo "name is empty" + exit 1 +fi + +if [ -z "$start" ]; then + echo "start is empty" + exit 1 +fi + + +DAEMON_PATH="/etc/systemd/system/${name}.service" + +cat < $DAEMON_PATH +[Unit] +Description = $name service. Created by https://github.com/igroman787/mypylib. +After = network.target + +[Service] +Type = simple +Restart = always +RestartSec = 30 +ExecStart = $start +ExecStopPost = $post +User = $user +Group = $group +LimitNOFILE = infinity +LimitNPROC = infinity +LimitMEMLOCK = infinity + +[Install] +WantedBy = multi-user.target +EOF + +chmod 664 $DAEMON_PATH +chmod +x $DAEMON_PATH +systemctl daemon-reload +systemctl enable ${name} diff --git a/scripts/jsonrpcinstaller.sh b/mytoninstaller/scripts/jsonrpcinstaller.sh old mode 100644 new mode 100755 similarity index 62% rename from scripts/jsonrpcinstaller.sh rename to mytoninstaller/scripts/jsonrpcinstaller.sh index a401eca3..e6338480 --- a/scripts/jsonrpcinstaller.sh +++ b/mytoninstaller/scripts/jsonrpcinstaller.sh @@ -15,6 +15,13 @@ do esac done +author=kdimentionaltree +repo=mtc-jsonrpc +branch=master + +echo "User: $user" +echo "Workdir: `pwd`" + # Цвета COLOR='\033[95m' ENDC='\033[0m' @@ -25,14 +32,17 @@ pip3 install Werkzeug json-rpc cloudscraper pyotp # Клонирование репозиториев с github.com echo -e "${COLOR}[2/4]${ENDC} Cloning github repository" +echo "https://github.com/${author}/${repo}.git -> ${branch}" + cd /usr/src/ rm -rf mtc-jsonrpc -git clone --recursive https://github.com/igroman787/mtc-jsonrpc.git +git clone --branch=${branch} --recursive https://github.com/${author}/${repo}.git # Прописать автозагрузку echo -e "${COLOR}[3/4]${ENDC} Add to startup" -cmd="from sys import path; path.append('/usr/src/mytonctrl/'); from mypylib.mypylib import *; Add2Systemd(name='mtc-jsonrpc', user='${user}', start='/usr/bin/python3 /usr/src/mtc-jsonrpc/mtc-jsonrpc.py')" -python3 -c "${cmd}" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +echo "Script dir: ${SCRIPT_DIR}" +${SCRIPT_DIR}/add2systemd.sh -n mtc-jsonrpc -s "/usr/bin/python3 /usr/src/mtc-jsonrpc/mtc-jsonrpc.py" -u ${user} -g ${user} systemctl restart mtc-jsonrpc # Выход из программы diff --git a/scripts/pytonv3installer.sh b/mytoninstaller/scripts/pytonv3installer.sh old mode 100644 new mode 100755 similarity index 76% rename from scripts/pytonv3installer.sh rename to mytoninstaller/scripts/pytonv3installer.sh index 0eb8d730..32010a4c --- a/scripts/pytonv3installer.sh +++ b/mytoninstaller/scripts/pytonv3installer.sh @@ -39,8 +39,9 @@ cd /usr/bin/ton && make tonlibjson # Прописать автозагрузку echo -e "${COLOR}[3/4]${ENDC} Add to startup" -cmd="from sys import path; path.append('/usr/src/mytonctrl/'); from mypylib.mypylib import *; Add2Systemd(name='pytonv3', user='${user}', workdir='/usr/src/pytonv3', start='/usr/bin/python3 -m pyTON --liteserverconfig /usr/bin/ton/local.config.json --libtonlibjson /usr/bin/ton/tonlib/libtonlibjson.so')" -python3 -c "${cmd}" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +echo "Script dir: ${SCRIPT_DIR}" +${SCRIPT_DIR}/add2systemd -n pytonv3 -s "/usr/bin/python3 -m pyTON --liteserverconfig /usr/bin/ton/local.config.json --libtonlibjson /usr/bin/ton/tonlib/libtonlibjson.so" -u ${user} -g ${user} systemctl restart pytonv3 # Конец diff --git a/mytoninstaller/scripts/tonhttpapiinstaller.sh b/mytoninstaller/scripts/tonhttpapiinstaller.sh new file mode 100755 index 00000000..c22b98e0 --- /dev/null +++ b/mytoninstaller/scripts/tonhttpapiinstaller.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# Проверить sudo +if [ "$(id -u)" != "0" ]; then + echo "Please run script as root" + exit 1 +fi + +# Get arguments +while getopts u: flag +do + case "${flag}" in + u) user=${OPTARG};; + esac +done + +# Цвета +COLOR='\033[92m' +ENDC='\033[0m' + +# Установка компонентов python3 +echo -e "${COLOR}[1/3]${ENDC} Installing required packages" +pip3 install -U ton-http-api + +# Установка модуля +echo -e "${COLOR}[2/3]${ENDC} Add to startup" +mkdir -p /var/ton-http-api/ton_keystore/ +chown -R $user /var/ton-http-api/ + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cmd="ton-http-api --port=8000 --logs-level=INFO --cdll-path=/usr/bin/ton/tonlib/libtonlibjson.so --liteserver-config /usr/bin/ton/local.config.json --tonlib-keystore=/var/ton-http-api/ton_keystore/ --parallel-requests-per-liteserver=1024" +${SCRIPT_DIR}/add2systemd.sh -n ton-http-api -s "${cmd}" -u ${user} -g ${user} +systemctl restart ton-http-api + +# Конец +echo -e "${COLOR}[3/3]${ENDC} TonHttpApi installation complete" +exit 0 diff --git a/mytoninstaller.py b/mytoninstaller/settings.py similarity index 61% rename from mytoninstaller.py rename to mytoninstaller/settings.py index 25f53d93..df75edc3 100644 --- a/mytoninstaller.py +++ b/mytoninstaller/settings.py @@ -1,249 +1,26 @@ -#!/usr/bin/env python3 -# -*- coding: utf_8 -*- - -import pwd -import random +import os +import os.path +import psutil +import subprocess import requests -from mypylib.mypylib import * -from mypyconsole.mypyconsole import * - -local = MyPyClass(__file__) -console = MyPyConsole() -defaultLocalConfigPath = "/usr/bin/ton/local.config.json" - - -def Init(): - local.db.config.isStartOnlyOneProcess = False - local.db.config.logLevel = "debug" - local.db.config.isIgnorLogWarning = True # disable warning - local.run() - local.db.config.isIgnorLogWarning = False # enable warning - - - # create variables - user = os.environ.get("USER", "root") - local.buffer.user = user - local.buffer.vuser = "validator" - local.buffer.cport = random.randint(2000, 65000) - local.buffer.lport = random.randint(2000, 65000) - - # Create user console - console.name = "MyTonInstaller" - console.color = console.RED - console.AddItem("status", Status, "Print TON component status") - console.AddItem("enable", Enable, "Enable some function: 'FN' - Full node, 'VC' - Validator console, 'LS' - Liteserver, 'DS' - DHT-Server, 'JR' - jsonrpc, 'PT' - pyTONv3. Example: 'enable FN'") - console.AddItem("update", Enable, "Update some function: 'JR' - jsonrpc. Example: 'update JR'") - console.AddItem("plsc", PrintLiteServerConfig, "Print LiteServer config") - console.AddItem("clcf", CreateLocalConfigFile, "CreateLocalConfigFile") - console.AddItem("drvcf", DRVCF, "Dangerous recovery validator config file") - console.AddItem("setwebpass", SetWebPassword, "Set a password for the web admin interface") - - Refresh() -#end define - -def Refresh(): - user = local.buffer.user - local.buffer.mconfig_path = "/home/{user}/.local/share/mytoncore/mytoncore.db".format(user=user) - if user == 'root': - local.buffer.mconfig_path = "/usr/local/bin/mytoncore/mytoncore.db" - #end if - - # create variables - bin_dir = "/usr/bin/" - src_dir = "/usr/src/" - ton_work_dir = "/var/ton-work/" - ton_bin_dir = bin_dir + "ton/" - ton_src_dir = src_dir + "ton/" - local.buffer.bin_dir = bin_dir - local.buffer.src_dir = src_dir - local.buffer.ton_work_dir = ton_work_dir - local.buffer.ton_bin_dir = ton_bin_dir - local.buffer.ton_src_dir = ton_src_dir - ton_db_dir = ton_work_dir + "db/" - keys_dir = ton_work_dir + "keys/" - local.buffer.ton_db_dir = ton_db_dir - local.buffer.keys_dir = keys_dir - local.buffer.ton_log_path = ton_work_dir + "log" - local.buffer.validator_app_path = ton_bin_dir + "validator-engine/validator-engine" - local.buffer.global_config_path = ton_bin_dir + "global.config.json" - local.buffer.vconfig_path = ton_db_dir + "config.json" -#end define - -def Status(args): - keys_dir = local.buffer.keys_dir - server_key = keys_dir + "server" - client_key = keys_dir + "client" - liteserver_key = keys_dir + "liteserver" - liteserver_pubkey = liteserver_key + ".pub" - - - fnStatus = os.path.isfile(local.buffer.vconfig_path) - mtcStatus = os.path.isfile(local.buffer.mconfig_path) - vcStatus = os.path.isfile(server_key) or os.path.isfile(client_key) - lsStatus = os.path.isfile(liteserver_pubkey) - - print("Full node status:", fnStatus) - print("Mytoncore status:", mtcStatus) - print("V.console status:", vcStatus) - print("Liteserver status:", lsStatus) -#end define - -def Enable(args): - name = args[0] - if name == "PT": - CreateLocalConfigFile(args) - args = ["python3", local.buffer.my_path, "-u", local.buffer.user, "-e", "enable{name}".format(name=name)] - run_as_root(args) -#end define - -def DRVCF(args): - args = ["python3", local.buffer.my_path, "-u", local.buffer.user, "-e", "drvcf"] - run_as_root(args) -#end define - -def get_own_ip(): - requests.packages.urllib3.util.connection.HAS_IPV6 = False - ip = requests.get("https://ifconfig.me/ip").text - return ip -#end define - -def GetLiteServerConfig(): - keys_dir = local.buffer.keys_dir - liteserver_key = keys_dir + "liteserver" - liteserver_pubkey = liteserver_key + ".pub" - result = Dict() - file = open(liteserver_pubkey, 'rb') - data = file.read() - file.close() - key = base64.b64encode(data[4:]) - ip = get_own_ip() - mconfig = GetConfig(path=local.buffer.mconfig_path) - result.ip = ip2int(ip) - result.port = mconfig.liteClient.liteServer.port - result.id = Dict() - result.id["@type"]= "pub.ed25519" - result.id.key= key.decode() - return result -#end define - -def GetInitBlock(): - from mytoncore import MyTonCore - ton = MyTonCore() - initBlock = ton.GetInitBlock() - return initBlock -#end define - -def CreateLocalConfig(initBlock, localConfigPath=defaultLocalConfigPath): - # dirty hack, but GetInitBlock() function uses the same technique - from mytoncore import hex2base64 - - # read global config file - file = open("/usr/bin/ton/global.config.json", 'rt') - text = file.read() - data = json.loads(text) - file.close() - - # edit config - liteServerConfig = GetLiteServerConfig() - data["liteservers"] = [liteServerConfig] - data["validator"]["init_block"]["seqno"] = initBlock["seqno"] - data["validator"]["init_block"]["root_hash"] = hex2base64(initBlock["rootHash"]) - data["validator"]["init_block"]["file_hash"] = hex2base64(initBlock["fileHash"]) - text = json.dumps(data, indent=4) - - # write local config file - file = open(localConfigPath, 'wt') - file.write(text) - file.close() - - # chown - user = local.buffer.user - args = ["chown", "-R", user + ':' + user, localConfigPath] - - print("Local config file created:", localConfigPath) -#end define - -def PrintLiteServerConfig(args): - liteServerConfig = GetLiteServerConfig() - text = json.dumps(liteServerConfig, indent=4) - print(text) -#end define - -def CreateLocalConfigFile(args): - initBlock = GetInitBlock() - initBlock_b64 = dict2b64(initBlock) - args = ["python3", local.buffer.my_path, "-u", local.buffer.user, "-e", "clc", "-i", initBlock_b64] - run_as_root(args) -#end define - -def Event(name): - if name == "enableFN": - FirstNodeSettings() - if name == "enableVC": - EnableValidatorConsole() - if name == "enableLS": - EnableLiteServer() - if name == "enableDS": - EnableDhtServer() - if name == "drvcf": - DangerousRecoveryValidatorConfigFile() - if name == "enableJR": - EnableJsonRpc() - if name == "enablePT": - EnablePytonv3() - if name == "clc": - ix = sys.argv.index("-i") - initBlock_b64 = sys.argv[ix+1] - initBlock = b642dict(initBlock_b64) - CreateLocalConfig(initBlock) -#end define - -def General(): - if "-u" in sys.argv: - ux = sys.argv.index("-u") - user = sys.argv[ux+1] - local.buffer.user = user - Refresh() - if "-e" in sys.argv: - ex = sys.argv.index("-e") - name = sys.argv[ex+1] - Event(name) - if "-m" in sys.argv: - mx = sys.argv.index("-m") - mode = sys.argv[mx+1] - if "-t" in sys.argv: - mx = sys.argv.index("-t") - telemetry = sys.argv[mx+1] - local.buffer.telemetry = Str2Bool(telemetry) - if "--dump" in sys.argv: - mx = sys.argv.index("--dump") - dump = sys.argv[mx+1] - local.buffer.dump = Str2Bool(dump) - #end if - - # Создать настройки для mytoncore.py - FirstMytoncoreSettings() - - if mode == "full": - FirstNodeSettings() - EnableValidatorConsole() - EnableLiteServer() - BackupVconfig() - BackupMconfig() - #end if - - # Создать символические ссылки - CreateSymlinks() - #end if -#end define - -def Str2Bool(str): - if str == "true": - return True - return False -#end define - -def FirstNodeSettings(): +import random +import json +import pkg_resources + +from mypylib.mypylib import ( + add2systemd, + get_dir_from_path, + run_as_root, + color_print, + ip2int, + Dict +) +from mytoninstaller.utils import StartValidator, StartMytoncore +from mytoninstaller.config import SetConfig, GetConfig, get_own_ip +from mytoncore.utils import hex2b64 + + +def FirstNodeSettings(local): local.add_log("start FirstNodeSettings fuction", "debug") # Создать переменные @@ -259,7 +36,7 @@ def FirstNodeSettings(): # Проверить конфигурацию if os.path.isfile(vconfig_path): - local.add_log("Validators config.json already exist. Break FirstNodeSettings fuction", "warning") + local.add_log(f"Validators config '{vconfig_path}' already exist. Break FirstNodeSettings fuction", "warning") return #end if @@ -295,7 +72,7 @@ def FirstNodeSettings(): subprocess.run(args) # Скачать дамп - DownloadDump() + DownloadDump(local) # chown 1 local.add_log("Chown ton-work dir", "debug") @@ -303,10 +80,11 @@ def FirstNodeSettings(): subprocess.run(args) # start validator - StartValidator() + StartValidator(local) #end define -def DownloadDump(): + +def DownloadDump(local): dump = local.buffer.dump if dump == False: return @@ -331,18 +109,25 @@ def DownloadDump(): os.system(cmd) #end define -def FirstMytoncoreSettings(): + +def FirstMytoncoreSettings(local): local.add_log("start FirstMytoncoreSettings fuction", "debug") user = local.buffer.user # Прописать mytoncore.py в автозагрузку - add2systemd(name="mytoncore", user=user, start="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py") + # add2systemd(name="mytoncore", user=user, start="/usr/bin/python3 /usr/src/mytonctrl/mytoncore.py") # TODO: fix path + add2systemd(name="mytoncore", user=user, start="/usr/bin/python3 -m mytoncore") # Проверить конфигурацию path = "/home/{user}/.local/share/mytoncore/mytoncore.db".format(user=user) + if os.path.isfile(path): + local.add_log(f"{path} already exist. Break FirstMytoncoreSettings fuction", "warning") + return + #end if + path2 = "/usr/local/bin/mytoncore/mytoncore.db" - if os.path.isfile(path) or os.path.isfile(path2): - local.add_log("mytoncore.db already exist. Break FirstMytoncoreSettings fuction", "warning") + if os.path.isfile(path2): + local.add_log(f"{path2}.db already exist. Break FirstMytoncoreSettings fuction", "warning") return #end if @@ -395,10 +180,10 @@ def FirstMytoncoreSettings(): subprocess.run(args) # start mytoncore - StartMytoncore() + StartMytoncore(local) #end define -def EnableValidatorConsole(): +def EnableValidatorConsole(local): local.add_log("start EnableValidatorConsole function", "debug") # Create variables @@ -417,8 +202,13 @@ def EnableValidatorConsole(): server_pubkey = server_key + ".pub" # Check if key exist - if os.path.isfile(server_key) or os.path.isfile(client_key): - local.add_log("Server or client key already exist. Break EnableValidatorConsole fuction", "warning") + if os.path.isfile(server_key): + local.add_log(f"Server key '{server_key}' already exist. Break EnableValidatorConsole fuction", "warning") + return + #end if + + if os.path.isfile(client_key): + local.add_log(f"Client key '{client_key}' already exist. Break EnableValidatorConsole fuction", "warning") return #end if @@ -468,7 +258,7 @@ def EnableValidatorConsole(): SetConfig(path=vconfig_path, data=vconfig) # restart validator - StartValidator() + StartValidator(local) # read mconfig mconfig_path = local.buffer.mconfig_path @@ -486,15 +276,16 @@ def EnableValidatorConsole(): SetConfig(path=mconfig_path, data=mconfig) # Подтянуть событие в mytoncore.py - cmd = "python3 {src_dir}mytonctrl/mytoncore.py -e \"enableVC\"".format(src_dir=src_dir) + # cmd = "python3 {srcDir}mytonctrl/mytoncore.py -e \"enableVC\"".format(srcDir=srcDir) + cmd = 'python3 -m mytoncore -e "enableVC"' args = ["su", "-l", user, "-c", cmd] subprocess.run(args) # restart mytoncore - StartMytoncore() + StartMytoncore(local) #end define -def EnableLiteServer(): +def EnableLiteServer(local): local.add_log("start EnableLiteServer function", "debug") # Create variables @@ -512,7 +303,7 @@ def EnableLiteServer(): # Check if key exist if os.path.isfile(liteserver_pubkey): - local.add_log("Liteserver key already exist. Break EnableLiteServer fuction", "warning") + local.add_log(f"Liteserver key '{liteserver_pubkey}' already exist. Break EnableLiteServer fuction", "warning") return #end if @@ -557,7 +348,7 @@ def EnableLiteServer(): SetConfig(path=vconfig_path, data=vconfig) # restart validator - StartValidator() + StartValidator(local) # edit mytoncore config file # read mconfig @@ -578,89 +369,114 @@ def EnableLiteServer(): SetConfig(path=mconfig_path, data=mconfig) # restart mytoncore - StartMytoncore() + StartMytoncore(local) #end define -def StartValidator(): - # restart validator - local.add_log("Start/restart validator service", "debug") - args = ["systemctl", "restart", "validator"] - subprocess.run(args) - # sleep 10 sec - local.add_log("sleep 10 sec", "debug") - time.sleep(10) -#end define +def EnableDhtServer(local): + local.add_log("start EnableDhtServer function", "debug") + vuser = local.buffer.vuser + ton_bin_dir = local.buffer.ton_bin_dir + globalConfigPath = local.buffer.global_config_path + dht_server = ton_bin_dir + "dht-server/dht-server" + generate_random_id = ton_bin_dir + "utils/generate-random-id" + tonDhtServerDir = "/var/ton-dht-server/" + tonDhtKeyringDir = tonDhtServerDir + "keyring/" -def StartMytoncore(): - # restart mytoncore - local.add_log("Start/restart mytoncore service", "debug") - args = ["systemctl", "restart", "mytoncore"] - subprocess.run(args) -#end define + # Проверить конфигурацию + dht_config_path = "/var/ton-dht-server/config.json" + if os.path.isfile(dht_config_path): + local.add_log(f"DHT-Server '{dht_config_path}' already exist. Break EnableDhtServer fuction", "warning") + return + #end if -def GetConfig(**kwargs): - path = kwargs.get("path") - file = open(path, 'rt') - text = file.read() - file.close() - config = Dict(json.loads(text)) - return config -#end define + # Подготовить папку + os.makedirs(tonDhtServerDir, exist_ok=True) -def SetConfig(**kwargs): - path = kwargs.get("path") - data = kwargs.get("data") + # Прописать автозагрузку + cmd = "{dht_server} -C {globalConfigPath} -D {tonDhtServerDir}" + cmd = cmd.format(dht_server=dht_server, globalConfigPath=globalConfigPath, tonDhtServerDir=tonDhtServerDir) + add2systemd(name="dht-server", user=vuser, start=cmd) - # write config + # Получить внешний ip адрес + ip = get_own_ip() + port = random.randint(2000, 65000) + addr = "{ip}:{port}".format(ip=ip, port=port) + + # Первый запуск + args = [dht_server, "-C", globalConfigPath, "-D", tonDhtServerDir, "-I", addr] + subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Получить вывод конфига + key = os.listdir(tonDhtKeyringDir)[0] + ip = ip2int(ip) + text = '{"@type": "adnl.addressList", "addrs": [{"@type": "adnl.address.udp", "ip": ' + str(ip) + ', "port": ' + str(port) + '}], "version": 0, "reinit_date": 0, "priority": 0, "expire_at": 0}' + args = [generate_random_id, "-m", "dht", "-k", tonDhtKeyringDir + key, "-a", text] + process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3) + output = process.stdout.decode("utf-8") + err = process.stderr.decode("utf-8") + if len(err) > 0: + raise Exception(err) + #end if + + data = json.loads(output) text = json.dumps(data, indent=4) - file = open(path, 'wt') - file.write(text) - file.close() -#end define + print(text) -def BackupVconfig(): - local.add_log("Backup validator config file 'config.json' to 'config.json.backup'", "debug") - vconfig_path = local.buffer.vconfig_path - backupPath = vconfig_path + ".backup" - args = ["cp", vconfig_path, backupPath] + # chown 1 + args = ["chown", "-R", vuser + ':' + vuser, tonDhtServerDir] subprocess.run(args) -#end define -def BackupMconfig(): - local.add_log("Backup mytoncore config file 'mytoncore.db' to 'mytoncore.db.backup'", "debug") - mconfig_path = local.buffer.mconfig_path - backupPath = mconfig_path + ".backup" - args = ["cp", mconfig_path, backupPath] + # start DHT-Server + args = ["systemctl", "restart", "dht-server"] subprocess.run(args) #end define -def GetPortsFromVconfig(): - vconfig_path = local.buffer.vconfig_path - # read vconfig - local.add_log("read vconfig", "debug") - vconfig = GetConfig(path=vconfig_path) +def EnableJsonRpc(local): + local.add_log("start EnableJsonRpc function", "debug") + user = local.buffer.user - # read mconfig - local.add_log("read mconfig", "debug") - mconfig_path = local.buffer.mconfig_path - mconfig = GetConfig(path=mconfig_path) + jsonrpcinstaller_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'jsonrpcinstaller.sh') + local.add_log(f"Running script: {jsonrpcinstaller_path}", "debug") + exitCode = run_as_root(["bash", jsonrpcinstaller_path, "-u", user]) # TODO: fix path + if exitCode == 0: + text = "EnableJsonRpc - {green}OK{endc}" + else: + text = "EnableJsonRpc - {red}Error{endc}" + color_print(text) +#end define - # edit mytoncore config file - local.add_log("edit mytoncore config file", "debug") - mconfig.liteClient.liteServer.port = mconfig.liteservers[0].port - mconfig.validatorConsole.addr = f"127.0.0.1:{mconfig.control[0].port}" - # write mconfig - local.add_log("write mconfig", "debug") - SetConfig(path=mconfig_path, data=mconfig) +def EnablePytonv3(local): + local.add_log("start EnablePytonv3 function", "debug") + user = local.buffer.user - # restart mytoncore - StartMytoncore() + pythonv3installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'pytonv3installer.sh') + local.add_log(f"Running script: {pythonv3installer_path}", "debug") + exitCode = run_as_root(["bash", pythonv3installer_path, "-u", user]) + if exitCode == 0: + text = "EnablePytonv3 - {green}OK{endc}" + else: + text = "EnablePytonv3 - {red}Error{endc}" + color_print(text) #end define -def DangerousRecoveryValidatorConfigFile(): + +def EnableTonHttpApi(local): + local.add_log("start EnablePytonv3 function", "debug") + user = local.buffer.user + + ton_http_api_installer_path = pkg_resources.resource_filename('mytoninstaller.scripts', 'tonhttpapiinstaller.sh') + exitCode = run_as_root(["bash", ton_http_api_installer_path, "-u", user]) + if exitCode == 0: + text = "EnableTonHttpApi - {green}OK{endc}" + else: + text = "EnableTonHttpApi - {red}Error{endc}" + color_print(text) + + +def DangerousRecoveryValidatorConfigFile(local): local.add_log("start DangerousRecoveryValidatorConfigFile function", "info") # install and import cryptography library @@ -865,21 +681,8 @@ def DangerousRecoveryValidatorConfigFile(): print("keys:", keys) #end define -def hex2b64(input): - hexBytes = bytes.fromhex(input) - b64Bytes = base64.b64encode(hexBytes) - b64String = b64Bytes.decode() - return b64String -#end define -def b642hex(input): - b64Bytes = input.encode() - hexBytes = base64.b64decode(b64Bytes) - hexString = hexBytes.hex() - return hexString -#end define - -def CreateSymlinks(): +def CreateSymlinks(local): local.add_log("start CreateSymlinks fuction", "debug") cport = local.buffer.cport @@ -889,7 +692,8 @@ def CreateSymlinks(): validator_console_file = "/usr/bin/validator-console" env_file = "/etc/environment" file = open(mytonctrl_file, 'wt') - file.write("/usr/bin/python3 /usr/src/mytonctrl/mytonctrl.py $@") + # file.write("/usr/bin/python3 /usr/src/mytonctrl/mytonctrl.py $@") # TODO: fix path + file.write("/usr/bin/python3 -m mytonctrl $@") # TODO: fix path file.close() file = open(fift_file, 'wt') file.write("/usr/bin/ton/crypto/fift $@") @@ -914,129 +718,3 @@ def CreateSymlinks(): file.write(fiftpath + '\n') file.close() #end define - -def EnableDhtServer(): - local.add_log("start EnableDhtServer function", "debug") - vuser = local.buffer.vuser - ton_bin_dir = local.buffer.ton_bin_dir - globalConfigPath = local.buffer.global_config_path - dht_server = ton_bin_dir + "dht-server/dht-server" - generate_random_id = ton_bin_dir + "utils/generate-random-id" - tonDhtServerDir = "/var/ton-dht-server/" - tonDhtKeyringDir = tonDhtServerDir + "keyring/" - - # Проверить конфигурацию - if os.path.isfile("/var/ton-dht-server/config.json"): - local.add_log("DHT-Server config.json already exist. Break EnableDhtServer fuction", "warning") - return - #end if - - # Подготовить папку - os.makedirs(tonDhtServerDir, exist_ok=True) - - # Прописать автозагрузку - cmd = "{dht_server} -C {globalConfigPath} -D {tonDhtServerDir}" - cmd = cmd.format(dht_server=dht_server, globalConfigPath=globalConfigPath, tonDhtServerDir=tonDhtServerDir) - add2systemd(name="dht-server", user=vuser, start=cmd) - - # Получить внешний ip адрес - ip = get_own_ip() - port = random.randint(2000, 65000) - addr = "{ip}:{port}".format(ip=ip, port=port) - - # Первый запуск - args = [dht_server, "-C", globalConfigPath, "-D", tonDhtServerDir, "-I", addr] - subprocess.run(args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - - # Получить вывод конфига - key = os.listdir(tonDhtKeyringDir)[0] - ip = ip2int(ip) - text = '{"@type": "adnl.addressList", "addrs": [{"@type": "adnl.address.udp", "ip": ' + str(ip) + ', "port": ' + str(port) + '}], "version": 0, "reinit_date": 0, "priority": 0, "expire_at": 0}' - args = [generate_random_id, "-m", "dht", "-k", tonDhtKeyringDir + key, "-a", text] - process = subprocess.run(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=3) - output = process.stdout.decode("utf-8") - err = process.stderr.decode("utf-8") - if len(err) > 0: - raise Exeption(err) - #end if - - data = json.loads(output) - text = json.dumps(data, indent=4) - print(text) - - # chown 1 - args = ["chown", "-R", vuser + ':' + vuser, tonDhtServerDir] - subprocess.run(args) - - # start DHT-Server - args = ["systemctl", "restart", "dht-server"] - subprocess.run(args) -#end define - -def SetWebPassword(args): - args = ["python3", "/usr/src/mtc-jsonrpc/mtc-jsonrpc.py", "-p"] - subprocess.run(args) -#end define - -def EnableJsonRpc(): - local.add_log("start EnableJsonRpc function", "debug") - user = local.buffer.user - exitCode = run_as_root(["bash", "/usr/src/mytonctrl/scripts/jsonrpcinstaller.sh", "-u", user]) - if exitCode == 0: - text = "EnableJsonRpc - {green}OK{endc}" - else: - text = "EnableJsonRpc - {red}Error{endc}" - color_print(text) -#end define - -def EnablePytonv3(): - local.add_log("start EnablePytonv3 function", "debug") - user = local.buffer.user - exitCode = run_as_root(["bash", "/usr/src/mytonctrl/scripts/pytonv3installer.sh", "-u", user]) - if exitCode == 0: - text = "EnablePytonv3 - {green}OK{endc}" - else: - text = "EnablePytonv3 - {red}Error{endc}" - color_print(text) -#end define - -def str2b64(s): - b = s.encode("utf-8") - b64 = base64.b64encode(b) - b64 = b64.decode("utf-8") - return b64 -#end define - -def b642str(b64): - b64 = b64.encode("utf-8") - b = base64.b64decode(b64) - s = b.decode("utf-8") - return s -#end define - -def dict2b64(d): - s = json.dumps(d) - b64 = str2b64(s) - return b64 -#end define - -def b642dict(b64): - s = b642str(b64) - d = json.loads(s) - return d -#end define - - - -### -### Start of the program -### - -if __name__ == "__main__": - Init() - if len(sys.argv) > 1: - General() - else: - console.Run() - local.exit() -#end if diff --git a/mytoninstaller/utils.py b/mytoninstaller/utils.py new file mode 100644 index 00000000..9b33b732 --- /dev/null +++ b/mytoninstaller/utils.py @@ -0,0 +1,33 @@ +import base64 +import json +import time +import subprocess + + +def GetInitBlock(): + from mytoncore import MyTonCore + + ton = MyTonCore(None) + initBlock = ton.GetInitBlock() + return initBlock +# end define + + +def StartValidator(local): + # restart validator + local.add_log("Start/restart validator service", "debug") + args = ["systemctl", "restart", "validator"] + subprocess.run(args) + + # sleep 10 sec + local.add_log("sleep 10 sec", "debug") + time.sleep(10) +# end define + + +def StartMytoncore(local): + # restart mytoncore + local.add_log("Start/restart mytoncore service", "debug") + args = ["systemctl", "restart", "mytoncore"] + subprocess.run(args) +# end define diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..bacf8a0e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +crc16 +requests +psutil +cryptography +fastcrc diff --git a/scripts/install.sh b/scripts/install.sh index 8f80eebb..5d6fc980 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,12 +1,21 @@ #!/bin/bash set -e -# Проверить sudo +# colors +COLOR='\033[92m' +ENDC='\033[0m' +mydir=`pwd` + +# check sudo permissions if [ "$(id -u)" != "0" ]; then - echo "Please run script as root" - exit 1 + echo "Please run script as root" + exit 1 fi +author="ton-blockchain" +repo="mytonctrl" +branch="master" + show_help_and_exit() { echo 'Supported argumets:' echo ' -m [lite|full] Choose installation mode' @@ -14,7 +23,10 @@ show_help_and_exit() { echo ' -t Disable telemetry' echo ' -i Ignore minimum reqiurements' echo ' -d Use pre-packaged dump. Reduces duration of initial synchronization.' - echo ' -h Show this help' + echo ' -a Set MyTonCtrl git repo author' + echo ' -r Set MyTonCtrl git repo' + echo ' -b Set MyTonCtrl git repo branch' + echo ' -h Show this help' exit } @@ -22,12 +34,14 @@ if [[ "${1-}" =~ ^-*h(elp)?$ ]]; then show_help_and_exit fi -# Get arguments +# node install parameters config="https://ton-blockchain.github.io/global.config.json" telemetry=true ignore=false dump=false -while getopts m:c:tidh flag + + +while getopts m:c:tida:r:b: flag do case "${flag}" in m) mode=${OPTARG};; @@ -35,23 +49,29 @@ do t) telemetry=false;; i) ignore=true;; d) dump=true;; - h) show_help_and_exit;; - *) + a) author=${OPTARG};; + r) repo=${OPTARG};; + b) branch=${OPTARG};; + h) show_help_and_exit;; + *) echo "Flag -${flag} is not recognized. Aborting" exit 1 ;; esac done - -# Проверка режима установки +# check installation mode if [ "${mode}" != "lite" ] && [ "${mode}" != "full" ]; then echo "Run script with flag '-m lite' or '-m full'" exit 1 fi -# Проверка мощностей +# check machine configuration +echo -e "${COLOR}[1/5]${ENDC} Checking system requirements" + cpus=$(lscpu | grep "CPU(s)" | head -n 1 | awk '{print $2}') -memory=$(grep MemTotal /proc/meminfo | awk '{print $2}') +memory=$(cat /proc/meminfo | grep MemTotal | awk '{print $2}') + +echo "This machine has ${cpus} CPUs and ${memory}KB of Memory" if [ "${mode}" = "lite" ] && [ "$ignore" = false ] && ([ "${cpus}" -lt 2 ] || [ "${memory}" -lt 2000000 ]); then echo "Insufficient resources. Requires a minimum of 2 processors and 2Gb RAM." exit 1 @@ -61,49 +81,53 @@ if [ "${mode}" = "full" ] && [ "$ignore" = false ] && ([ "${cpus}" -lt 8 ] || [ exit 1 fi -# Цвета -COLOR='\033[92m' -ENDC='\033[0m' - -# Начинаю установку mytonctrl -echo -e "${COLOR}[1/4]${ENDC} Starting installation MyTonCtrl" -mydir=$(pwd) - -# На OSX нет такой директории по-умолчанию, поэтому создаем... +echo -e "${COLOR}[2/5]${ENDC} Checking for required TON components" SOURCES_DIR=/usr/src BIN_DIR=/usr/bin + +# create dirs for OSX if [[ "$OSTYPE" =~ darwin.* ]]; then SOURCES_DIR=/usr/local/src BIN_DIR=/usr/local/bin mkdir -p ${SOURCES_DIR} fi -# Проверяю наличие компонентов TON -echo -e "${COLOR}[2/4]${ENDC} Checking for required TON components" +# check TON components file1=${BIN_DIR}/ton/crypto/fift file2=${BIN_DIR}/ton/lite-client/lite-client file3=${BIN_DIR}/ton/validator-engine-console/validator-engine-console -if [ -f "${file1}" ] && [ -f "${file2}" ] && [ -f "${file3}" ]; then - echo "TON exist" - cd $SOURCES_DIR - rm -rf $SOURCES_DIR/mytonctrl - git clone --recursive https://github.com/ton-blockchain/mytonctrl.git -else - rm -f toninstaller.sh - wget https://raw.githubusercontent.com/ton-blockchain/mytonctrl/master/scripts/toninstaller.sh - bash toninstaller.sh -c "${config}" - rm -f toninstaller.sh + +if [ ! -f "${file1}" ] || [ ! -f "${file2}" ] || [ ! -f "${file3}" ]; then + echo "TON does not exists, building" + wget https://raw.githubusercontent.com/${author}/${repo}/${branch}/scripts/ton_installer.sh -O /tmp/ton_installer.sh + bash /tmp/ton_installer.sh -c ${config} fi -# Запускаю установщик mytoninstaller.py -echo -e "${COLOR}[3/4]${ENDC} Launching the mytoninstaller.py" +# Cloning mytonctrl +echo -e "${COLOR}[3/5]${ENDC} Installing MyTonCtrl" +echo "https://github.com/${author}/${repo}.git -> ${branch}" + +cd $SOURCES_DIR +rm -rf $SOURCES_DIR/mytonctrl + +git clone https://github.com/${author}/${repo}.git ${repo} # TODO: return --recursive back when fix libraries +cd $SOURCES_DIR/${repo} +git checkout ${branch} +git submodule update --init --recursive +git config --global --add safe.directory $SOURCES_DIR/${repo} + +pip3 install -U . # TODO: make installation from git directly + +echo -e "${COLOR}[3/5]${ENDC} Running myton.installer" +# DEBUG + parent_name=$(ps -p $PPID -o comm=) user=$(whoami) if [ "$parent_name" = "sudo" ] || [ "$parent_name" = "su" ]; then user=$(logname) fi -python3 ${SOURCES_DIR}/mytonctrl/mytoninstaller.py -m ${mode} -u ${user} -t ${telemetry} --dump ${dump} +echo "User: $user" +python3 -m mytoninstaller -m ${mode} -u ${user} -t ${telemetry} --dump ${dump} -# Выход из программы echo -e "${COLOR}[4/4]${ENDC} Mytonctrl installation completed" exit 0 diff --git a/scripts/toninstaller.sh b/scripts/ton_installer.sh old mode 100755 new mode 100644 similarity index 66% rename from scripts/toninstaller.sh rename to scripts/ton_installer.sh index 807036e9..8ebab9ad --- a/scripts/toninstaller.sh +++ b/scripts/ton_installer.sh @@ -1,3 +1,4 @@ + #!/bin/bash set -e @@ -8,7 +9,7 @@ if [ "$(id -u)" != "0" ]; then fi # Get arguments -config="https://ton-blockchain.github.io/global.config.json" +config=https://ton-blockchain.github.io/global.config.json while getopts c: flag do case "${flag}" in @@ -32,55 +33,20 @@ fi # Установка требуемых пакетов echo -e "${COLOR}[1/6]${ENDC} Installing required packages" if [ "$OSTYPE" == "linux-gnu" ]; then - - # Determine if it is a CentOS system - if cat /etc/*release | grep ^NAME | grep CentOS ; then - echo "CentOS Linux detected." - yum update -y + if [ hash yum 2>/dev/null ]; then + echo "RHEL-based Linux detected." yum install -y epel-release - yum install -y git gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel g++ gcc libstdc++-devel zlib1g-dev libssl-dev which make gcc-c++ libstdc++-devel - - # Upgrade make and gcc in CentOS system - yum install centos-release-scl -y - yum install devtoolset-10 -y - echo "source /opt/rh/devtoolset-10/enable" >> /etc/bashrc - source /opt/rh/devtoolset-10/enable - - # Install the new version of CMake - yum remove cmake -y - wget https://cmake.org/files/v3.24/cmake-3.24.2.tar.gz - tar -zxvf cmake-3.24.2.tar.gz - cd cmake-3.24.2 && ./bootstrap && make -j -j$(nproc) && make install - yum remove cmake -y - ln -s /usr/local/bin/cmake /usr/bin/cmake - source ~/.bashrc - cmake --version - - # Install ninja - yum install -y ninja-build - - # Red Hat systems are not supported - elif cat /etc/*release | grep ^NAME | grep Red ; then - echo "Red Hat Linux detected." - echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." - exit 1 - - # Suse systems are not supported + dnf config-manager --set-enabled PowerTools + yum install -y git make cmake clang gflags gflags-devel zlib zlib-devel openssl-devel openssl-libs readline-devel libmicrohttpd python3 python3-pip python36-devel elif [ -f /etc/SuSE-release ]; then echo "Suse Linux detected." echo "This OS is not supported with this script at present. Sorry." echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." exit 1 - elif [ -f /etc/arch-release ]; then echo "Arch Linux detected." - pacman -Syuy --noconfirm - pacman -S --noconfirm git cmake clang gflags zlib openssl readline libmicrohttpd python python-pip - - # Install ninja - pacman -S --noconfirm ninja - + pacman -Syuy + pacman -S --noconfirm git make cmake clang gflags zlib openssl readline libmicrohttpd python python-pip elif [ -f /etc/debian_version ]; then echo "Ubuntu/Debian Linux detected." apt-get update @@ -95,8 +61,6 @@ if [ "$OSTYPE" == "linux-gnu" ]; then echo "Please refer to https://github.com/ton-blockchain/mytonctrl for setup information." exit 1 fi - -# Detected mac os system. elif [[ "$OSTYPE" =~ darwin.* ]]; then echo "Mac OS (Darwin) detected." if [ ! which brew >/dev/null 2>&1 ]; then @@ -108,34 +72,27 @@ elif [[ "$OSTYPE" =~ darwin.* ]]; then su $LOCAL_USERNAME -c "brew update" su $LOCAL_USERNAME -c "brew install openssl cmake llvm" - - # Install ninja - su $LOCAL_USERNAME -c "brew install ninja" - elif [ "$OSTYPE" == "freebsd"* ]; then echo "FreeBSD detected." echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/paritytech/substrate for setup information." + echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links exit 1 else echo "Unknown operating system." echo "This OS is not supported with this script at present. Sorry." - echo "Please refer to https://github.com/paritytech/substrate for setup information." + echo "Please refer to https://github.com/paritytech/substrate for setup information." # TODO: remove links exit 1 fi # Установка компонентов python3 -pip3 install psutil fastcrc requests +pip3 install psutil crc16 requests # Клонирование репозиториев с github.com echo -e "${COLOR}[2/6]${ENDC} Cloning github repository" cd $SOURCES_DIR rm -rf $SOURCES_DIR/ton -rm -rf $SOURCES_DIR/mytonctrl git clone --recursive https://github.com/ton-blockchain/ton.git -git clone --recursive https://github.com/ton-blockchain/mytonctrl.git git config --global --add safe.directory $SOURCES_DIR/ton -git config --global --add safe.directory $SOURCES_DIR/mytonctrl # Подготавливаем папки для компиляции echo -e "${COLOR}[3/6]${ENDC} Preparing for compilation" @@ -158,12 +115,12 @@ fi if [[ "$OSTYPE" =~ darwin.* ]]; then if [[ $(uname -p) == 'arm' ]]; then echo M1 - CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev -GNinja + CC="clang -mcpu=apple-a14" CXX="clang++ -mcpu=apple-a14" cmake $SOURCES_DIR/ton -DCMAKE_BUILD_TYPE=Release -DTON_ARCH= -Wno-dev else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton fi else - cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton -GNinja + cmake -DCMAKE_BUILD_TYPE=Release $SOURCES_DIR/ton fi # Компилируем из исходников @@ -180,7 +137,7 @@ else fi echo "use ${cpuNumber} cpus" -ninja -j ${cpuNumber} fift validator-engine lite-client validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy +make -j ${cpuNumber} fift validator-engine lite-client pow-miner validator-engine-console generate-random-id dht-server func tonlibjson rldp-http-proxy # Скачиваем конфигурационные файлы lite-client echo -e "${COLOR}[5/6]${ENDC} Downloading config files" diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index 87ef4a6b..a86575cc 100644 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -1,4 +1,10 @@ #!/bin/bash +full=true +while getopts f flag; do + case "${flag}" in + f) full=false + esac +done # Проверить sudo if [ "$(id -u)" != "0" ]; then @@ -26,11 +32,17 @@ rm -rf /etc/systemd/system/dht-server.service systemctl daemon-reload # Удаление файлов -rm -rf /usr/src/ton +if $full; then + echo "removing Ton node" + rm -rf /usr/src/ton + rm -rf /usr/bin/ton + rm -rf /var/ton-work + rm -rf /var/ton-dht-server +fi + rm -rf /usr/src/mytonctrl -rm -rf /usr/bin/ton -rm -rf /var/ton-work -rm -rf /var/ton-dht-server +rm -rf /usr/src/mtc-jsonrpc +rm -rf /usr/src/pytonv3 rm -rf /tmp/myton* rm -rf /usr/local/bin/mytoninstaller/ rm -rf /usr/local/bin/mytoncore/mytoncore.db @@ -38,10 +50,17 @@ rm -rf /home/${user}/.local/share/mytonctrl rm -rf /home/${user}/.local/share/mytoncore/mytoncore.db # Удаление ссылок -rm -rf /usr/bin/fift -rm -rf /usr/bin/liteclient -rm -rf /usr/bin/validator-console +if $full; then + echo "removing ton node" + rm -rf /usr/bin/fift + rm -rf /usr/bin/liteclient + rm -rf /usr/bin/validator-console +fi rm -rf /usr/bin/mytonctrl +# removing pip packages +pip3 uninstall -y myton mypylib mypyconsole +pip3 uninstall -y ton-http-api + # Конец echo -e "${COLOR}Uninstall Complete${ENDC}" diff --git a/scripts/upgrade.py b/scripts/upgrade.py deleted file mode 100644 index 9971694a..00000000 --- a/scripts/upgrade.py +++ /dev/null @@ -1,31 +0,0 @@ -# Говнокод ON -import os -from sys import path -from os.path import dirname as dir -path.append(dir(path[0])) -# Говнокод OFF - -from mypylib.mypylib import * - - -# validator.service -file = open("/etc/systemd/system/validator.service", 'rt') -text = file.read() -file.close() -lines = text.split('\n') - -for i in range(len(lines)): - line = lines[i] - if "ExecStart" not in line: - continue - if "ton-global.config.json" in line: - lines[i] += line.replace("validator-engine/ton-global.config.json", "global.config.json") -#end for - -text = "\n".join(lines) -file = open("/etc/systemd/system/validator.service", 'wt') -file.write(text) -file.close() - -args = ["systemctl", "daemon-reload"] -subprocess.run(args) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..b0b42415 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +from setuptools import setup, find_packages +from os.path import dirname, join + +with open(join(dirname(__file__), "README.md"), "r") as f: + long_description = f.read() + + +version = 'dev' + +setup( + author='igroman', + author_email='igroman', + name='myton', + version=version, + packages=find_packages('.', exclude=['tests']), + install_requires=[ + 'crc16', + 'requests', + 'psutil', + 'cryptography', + 'fastcrc', + ], + package_data={ + 'mytoninstaller.scripts': ['*.sh'], + 'mytonctrl': ['resources/*', 'scripts/*.sh'], + '': ['requirements.txt'], + }, + zip_safe=True, + python_requires='>=3.7', + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "License :: Other/Proprietary License", + "Topic :: Software Development :: Libraries" + ], + # url="https://github.com/toncenter/pytonlib", + description="MyTonCtrl", + long_description_content_type="text/markdown", + long_description=long_description, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/blocksScanner.py b/tests/blocksScanner.py index b8296746..88b012ff 100644 --- a/tests/blocksScanner.py +++ b/tests/blocksScanner.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*- -import sys -sys.path.append("/usr/src/mytonctrl/") -from mypylib.mypylib import bcolors, Sleep +from mypylib.mypylib import bcolors, Sleep, MyPyClass from mytoncore import MyTonCore, TonBlocksScanner def NewBlockReaction(block): @@ -19,7 +17,8 @@ def NewMessageReaction(message): #end define -ton = MyTonCore() +local = MyPyClass('./tests') +ton = MyTonCore(local) scanner = TonBlocksScanner(ton, nbr=NewBlockReaction, ntr=NewTransReaction, nmr=NewMessageReaction) scanner.Run() Sleep() diff --git a/tests/bounce.py b/tests/bounce.py index 79d60b32..08e5aa20 100644 --- a/tests/bounce.py +++ b/tests/bounce.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*-l -from sys import path -path.append("/usr/src/mytonctrl/") -from mytoncore import * +import time -Local = MyPyClass(__file__) -ton = MyTonCore() +from mypylib.mypylib import MyPyClass +from mytoncore import MyTonCore + +local = MyPyClass('./tests') +ton = MyTonCore(local) def Init(): wallets = list() - Local.buffer["wallets"] = wallets + local.buffer["wallets"] = wallets walletsNameList = ton.GetWalletsNameList() # Create tests wallet @@ -50,7 +51,7 @@ def Init(): buff_wallet = wallet buff_wallet.oldseqno = ton.GetSeqno(wallet) ton.MoveCoinsFromHW(wallet, [[testsWallet.addr, need]], wait=False) - Local.AddLog(testsWallet.name + " <<< " + str(wallet.subwallet)) + local.AddLog(testsWallet.name + " <<< " + str(wallet.subwallet)) if buff_wallet: ton.WaitTransaction(buff_wallet) #end for @@ -63,18 +64,18 @@ def Init(): if wallet.account.status == "uninit": wallet.oldseqno = ton.GetSeqno(wallet) ton.SendFile(wallet.bocFilePath) - Local.AddLog(str(wallet.subwallet) + " - OK") + local.AddLog(str(wallet.subwallet) + " - OK") ton.WaitTransaction(wallets[-1]) #end define def Work(): - wallets = Local.buffer["wallets"] + wallets = local.buffer["wallets"] destList = list() destList.append(["EQAY_2_A88HD43S96hbVGbCLB21e6_k1nbaqICwS3ZCrMBaZ", 2]) for wallet in wallets: wallet.oldseqno = ton.GetSeqno(wallet) ton.MoveCoinsFromHW(wallet, destList, wait=False) - Local.AddLog(str(wallet.subwallet) + " " + wallet.addr + " >>> ") + local.AddLog(str(wallet.subwallet) + " " + wallet.addr + " >>> ") ton.WaitTransaction(wallets[-1]) #end define @@ -83,7 +84,7 @@ def General(): while True: time.sleep(1) Work() - Local.AddLog("Work - OK") + local.AddLog("Work - OK") #end while #end define @@ -92,10 +93,10 @@ def General(): ### ### Start test ### -Local.Run() +local.Run() load = 200 -Local.StartCycle(General, sec=1) +local.StartCycle(General, sec=1) while True: time.sleep(60) #load += 10 diff --git a/tests/mg.py b/tests/mg.py index 536982b1..49877117 100644 --- a/tests/mg.py +++ b/tests/mg.py @@ -3,13 +3,13 @@ import sys import time -sys.path.append("/usr/src/mytonctrl/") + from mypylib.mypylib import bcolors, Sleep from mytoncore import MyTonCore from mypylib.mypylib import MyPyClass -ton = MyTonCore() local = MyPyClass(__file__) +ton = MyTonCore(local) def TestMoveCoins(wallet, dest, coins, **kwargs): start = time.time() diff --git a/tests/tpsLoad.py b/tests/tpsLoad.py index 8d44ae5c..3edb92dd 100644 --- a/tests/tpsLoad.py +++ b/tests/tpsLoad.py @@ -1,17 +1,21 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*-l -from sys import path -path.append("/usr/src/mytonctrl/") -from mytoncore import * +import time -Local = MyPyClass(__file__) -ton = MyTonCore() +from mypylib.mypylib import MyPyClass +from mytoncore import MyTonCore, Sleep + + +local = MyPyClass('./tests') +local.db["config"]["logLevel"] = "info" +load = 100 +ton = MyTonCore(local) def Init(): wallets = list() - Local.buffer["wallets"] = wallets + local.buffer["wallets"] = wallets walletsNameList = ton.GetWalletsNameList() # Create tests wallet @@ -50,7 +54,7 @@ def Init(): buff_wallet = wallet buff_wallet.oldseqno = ton.GetSeqno(wallet) ton.MoveGrams(wallet, testsWallet.addr, need, wait=False) - Local.AddLog(testsWallet.name + " <<< " + wallet.name) + local.AddLog(testsWallet.name + " <<< " + wallet.name) if buff_wallet: ton.WaitTransaction(buff_wallet, False) #end for @@ -63,12 +67,12 @@ def Init(): if wallet.account.status == "uninit": wallet.oldseqno = ton.GetSeqno(wallet) ton.SendFile(wallet.bocFilePath) - Local.AddLog(str(wallet.subwallet) + " - OK") + local.AddLog(str(wallet.subwallet) + " - OK") ton.WaitTransaction(wallets[-1]) #end define def Work(): - wallets = Local.buffer["wallets"] + wallets = local.buffer["wallets"] for i in range(load): if i + 1 == load: i = -1 @@ -78,7 +82,7 @@ def Work(): wallet2 = wallets[i+1] wallet1.oldseqno = ton.GetSeqno(wallet1) ton.MoveGrams(wallet1, wallet2.addr, 3.14, wait=False) - Local.AddLog(wallet1.name + " >>> " + wallet2.name) + local.AddLog(wallet1.name + " >>> " + wallet2.name) ton.WaitTransaction(wallets[-1]) #end define @@ -87,7 +91,7 @@ def General(): while True: time.sleep(1) Work() - Local.AddLog("Work - OK") + local.AddLog("Work - OK") #end while #end define @@ -96,13 +100,8 @@ def General(): ### ### Start test ### -Local = MyPyClass(__file__) -Local.db["config"]["logLevel"] = "info" -Local.Run() -ton = MyTonCore() -local.db["config"]["logLevel"] = "info" +local.Run() load = 100 - -Local.StartCycle(General, sec=1) +local.StartCycle(General, sec=1) Sleep() diff --git a/tests/tpsLoad2.py b/tests/tpsLoad2.py index 250c2800..b7f5f542 100644 --- a/tests/tpsLoad2.py +++ b/tests/tpsLoad2.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 # -*- coding: utf_8 -*-l -from sys import path -path.append("/usr/src/mytonctrl/") -from mytoncore import * +import time -Local = MyPyClass(__file__) -ton = MyTonCore() +from mypylib.mypylib import MyPyClass +from mytoncore import MyTonCore + +local = MyPyClass('./tests') +local.db["config"]["logLevel"] = "info" +load = 10 +ton = MyTonCore(local) def Init(): wallets = list() - Local.buffer["wallets"] = wallets + local.buffer["wallets"] = wallets walletsNameList = ton.GetWalletsNameList() # Create tests wallet @@ -50,7 +53,7 @@ def Init(): buff_wallet = wallet buff_wallet.oldseqno = ton.GetSeqno(wallet) ton.MoveGramsFromHW(wallet, [[testsWallet.addr, need]], wait=False) - Local.AddLog(testsWallet.name + " <<< " + str(wallet.subwallet)) + local.AddLog(testsWallet.name + " <<< " + str(wallet.subwallet)) if buff_wallet: ton.WaitTransaction(buff_wallet) #end for @@ -63,7 +66,7 @@ def Init(): if wallet.account.status == "uninit": wallet.oldseqno = ton.GetSeqno(wallet) ton.SendFile(wallet.bocFilePath) - Local.AddLog(str(wallet.subwallet) + " - OK") + local.AddLog(str(wallet.subwallet) + " - OK") ton.WaitTransaction(wallets[-1]) #end define @@ -75,7 +78,7 @@ def Work(): for wallet in wallets: wallet.oldseqno = ton.GetSeqno(wallet) ton.MoveGramsFromHW(wallet, destList, wait=False) - Local.AddLog(str(wallet.subwallet) + " " + wallet.addr + " >>> ") + local.AddLog(str(wallet.subwallet) + " " + wallet.addr + " >>> ") ton.WaitTransaction(wallets[-1]) #end define @@ -84,7 +87,7 @@ def General(): while True: time.sleep(1) Work() - Local.AddLog("Work - OK") + local.AddLog("Work - OK") #end while #end define @@ -93,15 +96,8 @@ def General(): ### ### Start test ### -Local = MyPyClass(__file__) -Local.db["config"]["logLevel"] = "info" -Local.Run() - -ton = MyTonCore() -local.db["config"]["logLevel"] = "info" -load = 10 - -Local.StartCycle(General, sec=1) +local.Run() +local.StartCycle(General, sec=1) while True: time.sleep(60) hour_str = time.strftime("%H")