diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a011dc45f..30869e93d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,7 @@ All notable changes to this project are documented in this file. - Disable ``prompt.py`` test cases due to high time consumption and unreliable results. - Migrate the existing test cases, which depend on BlockchainFixtureTestCase and WalletFixtureTestCase, to a privnet. Reduction of the fixtures' size to about 7MB. `#478 `_ - Ensure non-zero send value in prompt.py +- Update block importing and exporting functionality. - Add send-zero provision and improved test coverage to ``sendtoaddress`` diff --git a/neo/Core/Block.py b/neo/Core/Block.py index 65814e823..9df6a6340 100644 --- a/neo/Core/Block.py +++ b/neo/Core/Block.py @@ -123,11 +123,6 @@ def Size(self): return s def CalculatneNetFee(self, transactions): - # Transaction[] ts = transactions.Where(p= > p.Type != TransactionType.MinerTransaction & & p.Type != TransactionType.ClaimTransaction).ToArray(); - # Fixed8 amount_in = ts.SelectMany(p= > p.References.Values.Where(o= > o.AssetId == Blockchain.SystemCoin.Hash)).Sum(p= > p.Value); - # Fixed8 amount_out = ts.SelectMany(p= > p.Outputs.Where(o= > o.AssetId == Blockchain.SystemCoin.Hash)).Sum(p= > p.Value); - # Fixed8 amount_sysfee = ts.Sum(p= > p.SystemFee); - # return amount_in - amount_out - amount_sysfee; return 0 def TotalFees(self): @@ -151,6 +146,25 @@ def LoadTransactions(self): transactions = self.FullTransactions return transactions + def DeserializeForImport(self, reader): + """ + Deserialize full object. + + Args: + reader (neo.IO.BinaryReader): + """ + super(Block, self).Deserialize(reader) + + self.Transactions = [] + transaction_length = reader.ReadVarInt() + + for i in range(0, transaction_length): + tx = Transaction.DeserializeFrom(reader) + self.Transactions.append(tx) + + if len(self.Transactions) < 1: + raise Exception('Invalid format %s ' % self.Index) + def Deserialize(self, reader): """ Deserialize full object. @@ -220,6 +234,9 @@ def FromTrimmedData(byts): raise Exception("Could not find transaction!\n Are you running code against a valid Blockchain instance?\n Tests that accesses transactions or size of a block but inherit from NeoTestCase instead of BlockchainFixtureTestCase will not work.") tx_list.append(tx) + if len(tx_list) < 1: + raise Exception("Invalid block, no transactions found") + block.Transactions = tx_list StreamManager.ReleaseStream(ms) diff --git a/neo/Core/Helper.py b/neo/Core/Helper.py index 0a58322fa..c70b95528 100644 --- a/neo/Core/Helper.py +++ b/neo/Core/Helper.py @@ -69,7 +69,7 @@ def ToArray(value): value (neo.IO.Mixins.SerializableMixin): object extending SerializableMixin. Returns: - bytes: + bytes: hex formatted bytes """ ms = StreamManager.GetStream() writer = BinaryWriter(ms) @@ -81,6 +81,27 @@ def ToArray(value): return retVal + @staticmethod + def ToStream(value): + """ + Serialize the given `value` to a an array of bytes. + + Args: + value (neo.IO.Mixins.SerializableMixin): object extending SerializableMixin. + + Returns: + bytes: not hexlified + """ + ms = StreamManager.GetStream() + writer = BinaryWriter(ms) + + value.Serialize(writer) + + retVal = ms.getvalue() + StreamManager.ReleaseStream(ms) + + return retVal + @staticmethod def AddrStrToScriptHash(address): """ diff --git a/neo/Implementations/Notifications/LevelDB/NotificationDB.py b/neo/Implementations/Notifications/LevelDB/NotificationDB.py index 3c73bffea..df0d386dc 100644 --- a/neo/Implementations/Notifications/LevelDB/NotificationDB.py +++ b/neo/Implementations/Notifications/LevelDB/NotificationDB.py @@ -39,7 +39,6 @@ def instance(): if not NotificationDB.__instance: if settings.NOTIFICATION_DB_PATH: NotificationDB.__instance = NotificationDB(settings.notification_leveldb_path) -# logger.info("Created Notification DB At %s " % settings.NOTIFICATION_DB_PATH) else: logger.info("Notification DB Path not configured in settings") return NotificationDB.__instance diff --git a/neo/Prompt/Commands/Wallet.py b/neo/Prompt/Commands/Wallet.py index 09183fd5e..e8eb8234e 100644 --- a/neo/Prompt/Commands/Wallet.py +++ b/neo/Prompt/Commands/Wallet.py @@ -233,6 +233,7 @@ def ShowUnspentCoins(wallet, args): addr = None asset_type = None watch_only = 0 + do_count = False try: for item in args: if len(item) == 34: @@ -241,6 +242,8 @@ def ShowUnspentCoins(wallet, args): asset_type = get_asset_id(wallet, item) if item == '--watch': watch_only = 64 + elif item == '--count': + do_count = True except Exception as e: print("Invalid arguments specified") @@ -250,6 +253,11 @@ def ShowUnspentCoins(wallet, args): else: unspents = wallet.FindUnspentCoins(from_addr=addr, watch_only_val=watch_only) + if do_count: + print('\n-----------------------------------------------') + print('Total Unspent: %s' % len(unspents)) + return unspents + for unspent in unspents: print('\n-----------------------------------------------') print(json.dumps(unspent.ToJson(), indent=4)) @@ -268,11 +276,18 @@ def SplitUnspentCoin(wallet, args, prompt_passwd=True): :return: bool """ + + fee = Fixed8.Zero() + try: addr = wallet.ToScriptHash(args[0]) asset = get_asset_id(wallet, args[1]) index = int(args[2]) divisions = int(args[3]) + + if len(args) == 5: + fee = Fixed8.TryParse(args[4]) + except Exception as e: logger.info("Invalid arguments specified: %s " % e) return None @@ -285,7 +300,14 @@ def SplitUnspentCoin(wallet, args, prompt_passwd=True): outputs = split_to_vouts(asset, addr, unspentItem.Output.Value, divisions) + # subtract a fee from the first vout + if outputs[0].Value > fee: + outputs[0].Value -= fee + else: + raise Exception("Fee could not be subtracted from outputs.") + contract_tx = ContractTransaction(outputs=outputs, inputs=[unspentItem.Reference]) + ctx = ContractParametersContext(contract_tx) wallet.Sign(ctx) diff --git a/neo/SmartContract/ApplicationEngine.py b/neo/SmartContract/ApplicationEngine.py index 83d95defb..1fda17790 100644 --- a/neo/SmartContract/ApplicationEngine.py +++ b/neo/SmartContract/ApplicationEngine.py @@ -22,10 +22,9 @@ from neo.Core.State.AccountState import AccountState from neo.Core.State.ValidatorState import ValidatorState from neo.Core.State.StorageItem import StorageItem - from neo.Core.State.ContractState import ContractPropertyState from neo.SmartContract import TriggerType - +from neo.VM.InteropService import Array from neocore.UInt160 import UInt160 import datetime from neo.Settings import settings diff --git a/neo/SmartContract/SmartContractEvent.py b/neo/SmartContract/SmartContractEvent.py index a9d76e223..1368bc965 100644 --- a/neo/SmartContract/SmartContractEvent.py +++ b/neo/SmartContract/SmartContractEvent.py @@ -266,7 +266,10 @@ def SerializePayload(self, writer): writer.WriteUInt160(self.addr_from) writer.WriteUInt160(self.addr_to) - if self.Amount < 0xffffffffffffffff: + if self.Amount < 0: + logger.warn("Transfer Amount less than 0") + writer.WriteVarInt(0) + elif self.Amount < 0xffffffffffffffff: writer.WriteVarInt(self.amount) else: logger.warn("Writing Payload value amount greater than ulong long is not allowed. Setting to ulong long max") diff --git a/neo/VM/ExecutionEngine.py b/neo/VM/ExecutionEngine.py index 5984d85ac..d4e4bbb70 100644 --- a/neo/VM/ExecutionEngine.py +++ b/neo/VM/ExecutionEngine.py @@ -235,7 +235,6 @@ def ExecuteOp(self, opcode, context: ExecutionContext): script = self._Table.GetScript(UInt160(data=script_hash).ToBytes()) if script is None: - logger.error("Could not find script from script table: %s " % script_hash) return self.VM_FAULT_and_report(VMFault.INVALID_CONTRACT, script_hash) context_new = self.LoadScript(script) @@ -1011,32 +1010,6 @@ def StepInto(self): if self._exit_on_error: self._VMState |= VMState.FAULT - else: - logger.error(error_msg) - logger.exception(e) - - def StepOut(self): - self._VMState &= ~VMState.BREAK - count = self._InvocationStack.Count - - while self._VMState & VMState.HALT == 0 and \ - self._VMState & VMState.FAULT == 0 and \ - self._VMState & VMState.BREAK == 0 and \ - self._InvocationStack.Count > count: - self.StepInto() - - def StepOver(self): - if self._VMState & VMState.HALT > 0 or self._VMState & VMState.FAULT > 0: - return - - self._VMState &= ~VMState.BREAK - count = self._InvocationStack.Count - - while self._VMState & VMState.HALT == 0 and \ - self._VMState & VMState.FAULT == 0 and \ - self._VMState & VMState.BREAK == 0 and \ - self._InvocationStack.Count > count: - self.StepInto() def VM_FAULT_and_report(self, id, *args): self._VMState |= VMState.FAULT diff --git a/neo/VM/InteropService.py b/neo/VM/InteropService.py index eac50efa5..fada5fa37 100644 --- a/neo/VM/InteropService.py +++ b/neo/VM/InteropService.py @@ -534,12 +534,7 @@ def Register(self, method, func): def Invoke(self, method, engine): if method not in self._dictionary.keys(): - - logger.info("method %s not found in ->" % method) - for k, v in self._dictionary.items(): - logger.info("%s -> %s " % (k, v)) - return False - + logger.warn("method %s not found" % method) func = self._dictionary[method] # logger.info("[InteropService Method] %s " % func) return func(engine) diff --git a/neo/bin/export_blocks.py b/neo/bin/export_blocks.py index ad573dd44..657106764 100644 --- a/neo/bin/export_blocks.py +++ b/neo/bin/export_blocks.py @@ -6,6 +6,7 @@ import argparse from tqdm import trange import binascii +from neo.Core.Helper import Helper def main(): @@ -67,7 +68,14 @@ def main(): block = chain.GetBlockByHeight(index) block.LoadTransactions() - output = binascii.unhexlify(block.ToArray()) + + # make sure this block has transactions + # otherwise we will have a bad import + if len(block.Transactions) < 1: + raise Exception("Block %s has no transactions %s " % block.Index) + + output = Helper.ToStream(block) + output_length = len(output).to_bytes(4, 'little') file_out.write(output_length) file_out.write(output) diff --git a/neo/bin/import_blocks.py b/neo/bin/import_blocks.py index 21f23ffc2..2e9e950cf 100644 --- a/neo/bin/import_blocks.py +++ b/neo/bin/import_blocks.py @@ -4,13 +4,17 @@ from neo.Core.Block import Block from neo.IO.MemoryStream import MemoryStream from neo.Implementations.Blockchains.LevelDB.LevelDBBlockchain import LevelDBBlockchain +from neo.Implementations.Blockchains.LevelDB.DBPrefix import DBPrefix from neo.Settings import settings from neocore.IO.BinaryReader import BinaryReader +from neocore.IO.BinaryWriter import BinaryWriter +from neo.IO.MemoryStream import StreamManager, MemoryStream import argparse import os import shutil from tqdm import trange from prompt_toolkit import prompt +from neo.Implementations.Notifications.LevelDB.NotificationDB import NotificationDB def main(): @@ -29,6 +33,10 @@ def main(): parser.add_argument("-l", "--logevents", help="Log Smart Contract Events", default=False, action="store_true") + parser.add_argument("-n", "--notifications", help="Persist Notifications to database", default=False, action="store_true") + + parser.add_argument("-a", "--append", action="store_true", default=False, help="Append to current Block database") + args = parser.parse_args() if args.mainnet and args.config: @@ -52,59 +60,122 @@ def main(): raise Exception("Please specify an input path") file_path = args.input - with open(file_path, 'rb') as file_input: + append = False + store_notifications = False - total_blocks = int.from_bytes(file_input.read(4), 'little') + start_block = 0 - target_dir = os.path.join(settings.DATA_DIR_PATH, settings.LEVELDB_PATH) - notif_target_dir = os.path.join(settings.DATA_DIR_PATH, settings.NOTIFICATION_DB_PATH) + if args.append: + append = True + + if args.notifications: + store_notifications = True - print("Will import %s blocks to %s" % (total_blocks, target_dir)) - print("This will overwrite any data currently in %s and %s.\nType 'confirm' to continue" % (target_dir, notif_target_dir)) + header_hash_list = [] - confirm = prompt("[confirm]> ", is_password=False) - if not confirm == 'confirm': - print("Cancelled operation") - return False + with open(file_path, 'rb') as file_input: + + total_blocks_available = int.from_bytes(file_input.read(4), 'little') + + if total_blocks_available == 0: + total_blocks_available = int.from_bytes(file_input.read(4), 'little') - try: - if os.path.exists(target_dir): - shutil.rmtree(target_dir) - if os.path.exists(notif_target_dir): - shutil.rmtree(notif_target_dir) - except Exception as e: - print("Could not remove existing data %s " % e) - return False + total_blocks = total_blocks_available + if args.totalblocks and args.totalblocks < total_blocks and args.totalblocks > 0: + total_blocks = args.totalblocks + + target_dir = os.path.join(settings.DATA_DIR_PATH, settings.LEVELDB_PATH) + notif_target_dir = os.path.join(settings.DATA_DIR_PATH, settings.NOTIFICATION_DB_PATH) - # Instantiate the blockchain and subscribe to notifications - blockchain = LevelDBBlockchain(settings.chain_leveldb_path) - Blockchain.RegisterBlockchain(blockchain) + if append: + blockchain = LevelDBBlockchain(settings.chain_leveldb_path, skip_header_check=True) + Blockchain.RegisterBlockchain(blockchain) + + start_block = Blockchain.Default().Height + print("Starting import at %s " % start_block) + else: + print("Will import %s of %s blocks to %s" % (total_blocks, total_blocks_available, target_dir)) + print("This will overwrite any data currently in %s and %s.\nType 'confirm' to continue" % (target_dir, notif_target_dir)) + + confirm = prompt("[confirm]> ", is_password=False) + if not confirm == 'confirm': + print("Cancelled operation") + return False + + try: + if os.path.exists(target_dir): + shutil.rmtree(target_dir) + if os.path.exists(notif_target_dir): + shutil.rmtree(notif_target_dir) + except Exception as e: + print("Could not remove existing data %s " % e) + return False + + # Instantiate the blockchain and subscribe to notifications + blockchain = LevelDBBlockchain(settings.chain_leveldb_path) + Blockchain.RegisterBlockchain(blockchain) chain = Blockchain.Default() + if store_notifications: + NotificationDB.instance().start() + stream = MemoryStream() reader = BinaryReader(stream) block = Block() + length_ba = bytearray(4) for index in trange(total_blocks, desc='Importing Blocks', unit=' Block'): # set stream data - block_len = int.from_bytes(file_input.read(4), 'little') + file_input.readinto(length_ba) + block_len = int.from_bytes(length_ba, 'little') + reader.stream.write(file_input.read(block_len)) reader.stream.seek(0) # get block - block.Deserialize(reader) + block.DeserializeForImport(reader) + header_hash_list.append(block.Hash.ToBytes()) # add - if block.Index > 0: - chain.AddBlockDirectly(block) + if block.Index > start_block: + chain.AddBlockDirectly(block, do_persist_complete=store_notifications) # reset blockheader block._header = None + block.__hash = None # reset stream reader.stream.Cleanup() + print("Wrote blocks. Now writing headers") + + chain = Blockchain.Default() + + # reset header hash list + chain._db.delete(DBPrefix.IX_HeaderHashList) + + total = len(header_hash_list) + + chain._header_index = header_hash_list + + print("storing header hash list...") + + while total - 2000 >= chain._stored_header_count: + ms = StreamManager.GetStream() + w = BinaryWriter(ms) + headers_to_write = chain._header_index[chain._stored_header_count:chain._stored_header_count + 2000] + w.Write2000256List(headers_to_write) + out = ms.ToArray() + StreamManager.ReleaseStream(ms) + with chain._db.write_batch() as wb: + wb.put(DBPrefix.IX_HeaderHashList + chain._stored_header_count.to_bytes(4, 'little'), out) + + chain._stored_header_count += 2000 + + last_index = len(header_hash_list) + chain._db.put(DBPrefix.SYS_CurrentHeader, header_hash_list[-1] + last_index.to_bytes(4, 'little')) + print("Imported %s blocks to %s " % (total_blocks, target_dir)) diff --git a/neo/bin/prompt.py b/neo/bin/prompt.py index 0a2dc1457..2911a3bec 100755 --- a/neo/bin/prompt.py +++ b/neo/bin/prompt.py @@ -178,11 +178,6 @@ def __init__(self, history_filename=None): def get_bottom_toolbar(self, cli=None): out = [] try: - # Note: not sure if prompt-toolkit still supports foreground colors, couldn't get it to work - # out = [("class:command", '[%s] Progress: ' % settings.net_name), - # ("class:number", str(Blockchain.Default().Height + 1)), - # ("class:neo", '/'), - # ("class:number", str(Blockchain.Default().HeaderHeight + 1))] return "[%s] Progress: %s/%s" % (settings.net_name, str(Blockchain.Default().Height + 1), str(Blockchain.Default().HeaderHeight + 1))