New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[FEATURE] Memory usage improvements #379
Comments
Indexer batch profilingResultIt seems reasonable that indexer batch would use 0.6MB〜0.8MB of memory per a Token. Content
Additional Information
Snippets to reproduceimport sys
from web3 import Web3
from web3.contract import Contract
web3 = Web3(Web3.HTTPProvider("http:localhost:8545"))
abi = """[{"inputs": [{"internalType": "string", "name": "_name", "type": "string"}, {"internalType": "string", "name": "_symbol", "type": "string"}, {"internalType": "uint256", "name": "_issuePrice", "type": "uint256"}, {"internalType": "uint256", "name": "_totalSupply", "type": "uint256"}, {"internalType": "uint256", "name": "_dividends", "type": "uint256"}, {"internalType": "string", "name": "_dividendRecordDate", "type": "string"}, {"internalType": "string", "name": "_dividendPaymentDate", "type": "string"}, {"internalType": "string", "name": "_cancellationDate", "type": "string"}, {"internalType": "uint256", "name": "_principalValue", "type": "uint256"}], "stateMutability": "nonpayable", "type": "constructor"}, {"anonymous": false, "inputs": [{"indexed": false, "internalType": "address", "name": "accountAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "amount", "type": "uint256"}], "name": "Allot", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": false, "internalType": "address", "name": "accountAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "amount", "type": "uint256"}, {"indexed": false, "internalType": "string", "name": "data", "type": "string"}], "name": "ApplyForOffering", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "uint256", "name": "index", "type": "uint256"}, {"indexed": false, "internalType": "address", "name": "from", "type": "address"}, {"indexed": false, "internalType": "address", "name": "to", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "value", "type": "uint256"}, {"indexed": false, "internalType": "string", "name": "data", "type": "string"}], "name": "ApplyForTransfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "uint256", "name": "index", "type": "uint256"}, {"indexed": false, "internalType": "address", "name": "from", "type": "address"}, {"indexed": false, "internalType": "address", "name": "to", "type": "address"}, {"indexed": false, "internalType": "string", "name": "data", "type": "string"}], "name": "ApproveTransfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": false, "internalType": "address", "name": "lockAddress", "type": "address"}, {"indexed": false, "internalType": "bool", "name": "auth", "type": "bool"}], "name": "AuthorizeLockAddress", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "uint256", "name": "index", "type": "uint256"}, {"indexed": false, "internalType": "address", "name": "from", "type": "address"}, {"indexed": false, "internalType": "address", "name": "to", "type": "address"}, {"indexed": false, "internalType": "string", "name": "data", "type": "string"}], "name": "CancelTransfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": false, "internalType": "uint256", "name": "dividends", "type": "uint256"}, {"indexed": false, "internalType": "string", "name": "dividendRecordDate", "type": "string"}, {"indexed": false, "internalType": "string", "name": "dividendPaymentDate", "type": "string"}], "name": "ChangeDividendInformation", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "bool", "name": "status", "type": "bool"}], "name": "ChangeOfferingStatus", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "bool", "name": "status", "type": "bool"}], "name": "ChangeStatus", "type": "event"}, {"anonymous": false, "inputs": [], "name": "ChangeToCanceled", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": false, "internalType": "bool", "name": "required", "type": "bool"}], "name": "ChangeTransferApprovalRequired", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "from", "type": "address"}, {"indexed": true, "internalType": "address", "name": "targetAddress", "type": "address"}, {"indexed": true, "internalType": "address", "name": "lockAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "amount", "type": "uint256"}], "name": "Issue", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "accountAddress", "type": "address"}, {"indexed": true, "internalType": "address", "name": "lockAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "value", "type": "uint256"}], "name": "Lock", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "previousOwner", "type": "address"}, {"indexed": true, "internalType": "address", "name": "newOwner", "type": "address"}], "name": "OwnershipTransferred", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "from", "type": "address"}, {"indexed": true, "internalType": "address", "name": "targetAddress", "type": "address"}, {"indexed": true, "internalType": "address", "name": "lockAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "amount", "type": "uint256"}], "name": "Redeem", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "from", "type": "address"}, {"indexed": true, "internalType": "address", "name": "to", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "value", "type": "uint256"}], "name": "Transfer", "type": "event"}, {"anonymous": false, "inputs": [{"indexed": true, "internalType": "address", "name": "accountAddress", "type": "address"}, {"indexed": true, "internalType": "address", "name": "lockAddress", "type": "address"}, {"indexed": false, "internalType": "address", "name": "recipientAddress", "type": "address"}, {"indexed": false, "internalType": "uint256", "name": "value", "type": "uint256"}], "name": "Unlock", "type": "event"}, {"inputs": [{"internalType": "address", "name": "_accountAddress", "type": "address"}, {"internalType": "uint256", "name": "_amount", "type": "uint256"}], "name": "allot", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "", "type": "address"}], "name": "applicationsForOffering", "outputs": [{"internalType": "uint256", "name": "applicationAmount", "type": "uint256"}, {"internalType": "uint256", "name": "allotmentAmount", "type": "uint256"}, {"internalType": "string", "name": "data", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "name": "applicationsForTransfer", "outputs": [{"internalType": "address", "name": "from", "type": "address"}, {"internalType": "address", "name": "to", "type": "address"}, {"internalType": "uint256", "name": "amount", "type": "uint256"}, {"internalType": "bool", "name": "valid", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "_amount", "type": "uint256"}, {"internalType": "string", "name": "_data", "type": "string"}], "name": "applyForOffering", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_to", "type": "address"}, {"internalType": "uint256", "name": "_value", "type": "uint256"}, {"internalType": "string", "name": "_data", "type": "string"}], "name": "applyForTransfer", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "_index", "type": "uint256"}, {"internalType": "string", "name": "_data", "type": "string"}], "name": "approveTransfer", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_lockAddress", "type": "address"}, {"internalType": "bool", "name": "_auth", "type": "bool"}], "name": "authorizeLockAddress", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "", "type": "address"}], "name": "authorizedLockAddress", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_address", "type": "address"}], "name": "balanceOf", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "", "type": "address"}], "name": "balances", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address[]", "name": "_toList", "type": "address[]"}, {"internalType": "uint256[]", "name": "_valueList", "type": "uint256[]"}], "name": "bulkTransfer", "outputs": [{"internalType": "bool", "name": "success", "type": "bool"}], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "_index", "type": "uint256"}, {"internalType": "string", "name": "_data", "type": "string"}], "name": "cancelTransfer", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "cancellationDate", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "bool", "name": "_isOffering", "type": "bool"}], "name": "changeOfferingStatus", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "changeToCanceled", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "contactInformation", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "dividendInformation", "outputs": [{"internalType": "uint256", "name": "dividends", "type": "uint256"}, {"internalType": "string", "name": "dividendRecordDate", "type": "string"}, {"internalType": "string", "name": "dividendPaymentDate", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "isCanceled", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "isOffering", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_target_address", "type": "address"}, {"internalType": "address", "name": "_locked_address", "type": "address"}, {"internalType": "uint256", "name": "_amount", "type": "uint256"}], "name": "issueFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "issuePrice", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_lockAddress", "type": "address"}, {"internalType": "uint256", "name": "_value", "type": "uint256"}], "name": "lock", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}], "name": "locked", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_lockAddress", "type": "address"}, {"internalType": "address", "name": "_accountAddress", "type": "address"}], "name": "lockedOf", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "memo", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "name", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "offeringStatus", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "owner", "outputs": [{"internalType": "address", "name": "", "type": "address"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "", "type": "address"}], "name": "pendingTransfer", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "personalInfoAddress", "outputs": [{"internalType": "address", "name": "", "type": "address"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "principalValue", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "privacyPolicy", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_target_address", "type": "address"}, {"internalType": "address", "name": "_locked_address", "type": "address"}, {"internalType": "uint256", "name": "_amount", "type": "uint256"}], "name": "redeemFrom", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "uint8", "name": "", "type": "uint8"}], "name": "referenceUrls", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "string", "name": "_cancellationDate", "type": "string"}], "name": "setCancellationDate", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "string", "name": "_contactInformation", "type": "string"}], "name": "setContactInformation", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "_dividends", "type": "uint256"}, {"internalType": "string", "name": "_dividendRecordDate", "type": "string"}, {"internalType": "string", "name": "_dividendPaymentDate", "type": "string"}], "name": "setDividendInformation", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "string", "name": "_memo", "type": "string"}], "name": "setMemo", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_address", "type": "address"}], "name": "setPersonalInfoAddress", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "uint256", "name": "_principalValue", "type": "uint256"}], "name": "setPrincipalValue", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "string", "name": "_privacyPolicy", "type": "string"}], "name": "setPrivacyPolicy", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "bool", "name": "_status", "type": "bool"}], "name": "setStatus", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_exchange", "type": "address"}], "name": "setTradableExchange", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "bool", "name": "_required", "type": "bool"}], "name": "setTransferApprovalRequired", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "bool", "name": "_transferable", "type": "bool"}], "name": "setTransferable", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "status", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "symbol", "outputs": [{"internalType": "string", "name": "", "type": "string"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "totalSupply", "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], "stateMutability": "view", "type": "function"}, {"inputs": [], "name": "tradableExchange", "outputs": [{"internalType": "address", "name": "", "type": "address"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_to", "type": "address"}, {"internalType": "uint256", "name": "_value", "type": "uint256"}], "name": "transfer", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "transferApprovalRequired", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_from", "type": "address"}, {"internalType": "address", "name": "_to", "type": "address"}, {"internalType": "uint256", "name": "_value", "type": "uint256"}], "name": "transferFrom", "outputs": [{"internalType": "bool", "name": "success", "type": "bool"}], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [{"internalType": "address", "name": "newOwner", "type": "address"}], "name": "transferOwnership", "outputs": [], "stateMutability": "nonpayable", "type": "function"}, {"inputs": [], "name": "transferable", "outputs": [{"internalType": "bool", "name": "", "type": "bool"}], "stateMutability": "view", "type": "function"}, {"inputs": [{"internalType": "address", "name": "_accountAddress", "type": "address"}, {"internalType": "address", "name": "_recipientAddress", "type": "address"}, {"internalType": "uint256", "name": "_value", "type": "uint256"}], "name": "unlock", "outputs": [], "stateMutability": "nonpayable", "type": "function"}]"""
target_address = "0x6E18c885c625e5e2c47dcC9C73bE69646E884Ae5"
class TestProcessor:
def contract_factory(self, contract_address: str) -> Contract:
return web3.eth.contract(address=contract_address, abi=abi)
def contract_factory_with_delete(self, contract_address: str) -> Contract:
contract = web3.eth.contract(address=contract_address, abi=abi)
contract.functions = None
contract.caller = None
contract.abi = None
return contract
def bulk_create_contract_object(self, with_delete: bool):
self.contract_list = []
if with_delete:
for _ in range(250):
contract = self.contract_factory_with_delete(target_address)
self.contract_list.append(contract)
else:
for _ in range(250):
contract = self.contract_factory(target_address)
self.contract_list.append(contract)
def bulk_create_loop():
test_processor = TestProcessor()
with_delete = False
if len(sys.argv) > 1 and "with_delete" in sys.argv:
with_delete = True
for _ in range(20):
test_processor.bulk_create_contract_object(with_delete=with_delete)
if __name__ == "__main__":
bulk_create_loop() |
1. 250 call
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Is your feature request related to a problem? Please describe.
The amount of memory resources that indexer batch uses would be high.
Since the configuration provides the option to start them with a single container,
it is possible to run out of memory up to the memory usage limit of the container.
Reducing memory usage is required.
Describe the solution you'd like
Currently no. We would like to investigate memory usage and clear what potential memory savings are possible.
The text was updated successfully, but these errors were encountered: