eliminate use of global vars g.proto, g.coin, g.rpc and others

This patch eliminates nearly all the global variables that changed during the
execution of scripts.  With a few minor exceptions, global vars are now used
only during initialization or reserved for cfg file / cmdline options and other
unchanging values.

The result is a code base that's much more maintainable and extensible and less
error-prone.  The autosigning code, which supports signing of transactions for
multiple protocols and networks, has been greatly simplified.

Doing away with globals required many changes throughout the code base, and
other related (and not so related) changes and cleanups were made along the
way, resulting in an enormous patch.

Additional code changes include:

    - tx.py: complete reorganization of TX classes and use of nesting

    - protocol.py: separation of Regtest and Testnet into distinct subclasses
      with separate address and transaction files and file extensions

    - new module help.py for the help notes, loaded on demand

    - addr.py: rewrite of the address file label parsing code

    - tx.py,tw.py: use of generators to create formatted text

User-visible changes include:

    - importing of addresses for tokens not yet in the user's tracking wallet
      is now performed with the `--token-addr` option instead of `--token`

Testing:

    Testing this patch requires a full run of the test suite as described on the
    Test-Suite wiki page.

mmgen/altcoins/eth/contract.py

@@ -37,13 +37,13 @@
 def parse_abi(s):
 	return [s[:8]] + [s[8+x*64:8+(x+1)*64] for x in range(len(s[8:])//64)]
 
-def create_method_id(sig): return keccak_256(sig.encode()).hexdigest()[:8]
+def create_method_id(sig):
+	return keccak_256(sig.encode()).hexdigest()[:8]
 
 class TokenBase(MMGenObject): # ERC20
 
-	@staticmethod
-	def transferdata2sendaddr(data): # online
-		return CoinAddr(parse_abi(data)[1][-40:])
+	def transferdata2sendaddr(self,data): # online
+		return CoinAddr(self.proto,parse_abi(data)[1][-40:])
 
 	def transferdata2amt(self,data): # online
 		return ETHAmt(int(parse_abi(data)[-1],16) * self.base_unit)
@@ -52,7 +52,7 @@ async def do_call(self,method_sig,method_args='',toUnit=False):
 		data = create_method_id(method_sig) + method_args
 		if g.debug:
 			msg('ETH_CALL {}:  {}'.format(method_sig,'\n  '.join(parse_abi(data))))
-		ret = await g.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data })
+		ret = await self.rpc.call('eth_call',{ 'to': '0x'+self.addr, 'data': '0x'+data })
 		if toUnit:
 			return int(ret,16) * self.base_unit
 		else:
@@ -91,7 +91,7 @@ async def info(self):
 						'total supply:',   await self.get_total_supply())
 
 	async def code(self):
-		return (await g.rpc.call('eth_getCode','0x'+self.addr))[2:]
+		return (await self.rpc.call('eth_getCode','0x'+self.addr))[2:]
 
 	def create_data(self,to_addr,amt,method_sig='transfer(address,uint256)',from_addr=None):
 		from_arg = from_addr.rjust(64,'0') if from_addr else ''
@@ -114,14 +114,13 @@ async def txsign(self,tx_in,key,from_addr,chain_id=None):
 		from .pyethereum.transactions import Transaction
 
 		if chain_id is None:
-			chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in g.rpc.caps]
-			chain_id = int(await g.rpc.call(chain_id_method),16)
+			chain_id_method = ('parity_chainId','eth_chainId')['eth_chainId' in self.rpc.caps]
+			chain_id = int(await self.rpc.call(chain_id_method),16)
 		tx = Transaction(**tx_in).sign(key,chain_id)
 		hex_tx = rlp.encode(tx).hex()
 		coin_txid = CoinTxID(tx.hash.hex())
 		if tx.sender.hex() != from_addr:
-			m = "Sender address '{}' does not match address of key '{}'!"
-			die(3,m.format(from_addr,tx.sender.hex()))
+			die(3,f'Sender address {from_addr!r} does not match address of key {tx.sender.hex()!r}!')
 		if g.debug:
 			msg('TOKEN DATA:')
 			pp_msg(tx.to_dict())
@@ -131,7 +130,7 @@ async def txsign(self,tx_in,key,from_addr,chain_id=None):
 # The following are used for token deployment only:
 
 	async def txsend(self,hex_tx):
-		return (await g.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1)
+		return (await self.rpc.call('eth_sendRawTransaction','0x'+hex_tx)).replace('0x','',1)
 
 	async def transfer(   self,from_addr,to_addr,amt,key,start_gas,gasPrice,
 					method_sig='transfer(address,uint256)',
@@ -140,28 +139,32 @@ async def transfer(   self,from_addr,to_addr,amt,key,start_gas,gasPrice,
 		tx_in = self.make_tx_in(
 					from_addr,to_addr,amt,
 					start_gas,gasPrice,
-					nonce = int(await g.rpc.call('parity_nextNonce','0x'+from_addr),16),
+					nonce = int(await self.rpc.call('parity_nextNonce','0x'+from_addr),16),
 					method_sig = method_sig,
 					from_addr2 = from_addr2 )
 		(hex_tx,coin_txid) = await self.txsign(tx_in,key,from_addr)
 		return await self.txsend(hex_tx)
 
 class Token(TokenBase):
 
-	def __init__(self,addr,decimals):
-		self.addr = TokenAddr(addr)
+	def __init__(self,proto,addr,decimals,rpc=None):
+		self.proto = proto
+		self.addr = TokenAddr(proto,addr)
 		assert isinstance(decimals,int),f'decimals param must be int instance, not {type(decimals)}'
 		self.decimals = decimals
 		self.base_unit = Decimal('10') ** -self.decimals
+		self.rpc = rpc
 
 class TokenResolve(TokenBase,metaclass=aInitMeta):
 
-	def __init__(self,addr):
+	def __init__(self,*args,**kwargs):
 		return super().__init__()
 
-	async def __ainit__(self,addr):
-		self.addr = TokenAddr(addr)
+	async def __ainit__(self,proto,rpc,addr):
+		self.proto = proto
+		self.rpc = rpc
+		self.addr = TokenAddr(proto,addr)
 		decimals = await self.get_decimals() # requires self.addr!
 		if not decimals:
 			raise TokenNotInBlockchain(f'Token {addr!r} not in blockchain')
-		Token.__init__(self,addr,decimals)
+		Token.__init__(self,proto,addr,decimals,rpc)

mmgen/altcoins/eth/tw.py

@@ -21,7 +21,7 @@
 """
 
 from mmgen.common import *
-from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id,MMGenListItem,ListItemAttr,ImmutableAttr
+from mmgen.obj import ETHAmt,TwLabel,is_coin_addr,is_mmgen_id,ListItemAttr,ImmutableAttr
 from mmgen.tw import TrackingWallet,TwAddrList,TwUnspentOutputs,TwGetBalance
 from mmgen.addr import AddrData,TwAddrData
 from .contract import Token,TokenResolve
@@ -36,7 +36,7 @@ async def is_in_wallet(self,addr):
 		return addr in self.data_root
 
 	def init_empty(self):
-		self.data = { 'coin': g.coin, 'accounts': {}, 'tokens': {} }
+		self.data = { 'coin': self.proto.coin, 'accounts': {}, 'tokens': {} }
 
 	def upgrade_wallet_maybe(self):
 
@@ -49,7 +49,7 @@ def upgrade_wallet_maybe(self):
 				import json
 				self.data['accounts'] = json.loads(self.orig_data)
 			if not 'coin' in self.data:
-				self.data['coin'] = g.coin
+				self.data['coin'] = self.proto.coin
 			upgraded = True
 
 		def have_token_params_fields():
@@ -75,7 +75,7 @@ def add_token_params_fields():
 			msg('{} upgraded successfully!'.format(self.desc))
 
 	async def rpc_get_balance(self,addr):
-		return ETHAmt(int(await g.rpc.call('eth_getBalance','0x'+addr),16),'wei')
+		return ETHAmt(int(await self.rpc.call('eth_getBalance','0x'+addr),16),'wei')
 
 	@write_mode
 	async def batch_import_address(self,args_list):
@@ -97,17 +97,17 @@ async def import_address(self,addr,label,foo):
 	async def remove_address(self,addr):
 		r = self.data_root
 
-		if is_coin_addr(addr):
+		if is_coin_addr(self.proto,addr):
 			have_match = lambda k: k == addr
-		elif is_mmgen_id(addr):
+		elif is_mmgen_id(self.proto,addr):
 			have_match = lambda k: r[k]['mmid'] == addr
 		else:
 			die(1,f'{addr!r} is not an Ethereum address or MMGen ID')
 
 		for k in r:
 			if have_match(k):
 				# return the addr resolved to mmid if possible
-				ret = r[k]['mmid'] if is_mmgen_id(r[k]['mmid']) else addr
+				ret = r[k]['mmid'] if is_mmgen_id(self.proto,r[k]['mmid']) else addr
 				del r[k]
 				self.write()
 				return ret
@@ -152,30 +152,33 @@ class EthereumTokenTrackingWallet(EthereumTrackingWallet):
 	symbol = None
 	cur_eth_balances = {}
 
-	async def __ainit__(self,mode='r'):
-		await super().__ainit__(mode=mode)
+	async def __ainit__(self,proto,mode='r',token_addr=None):
+		await super().__ainit__(proto,mode=mode)
 
 		for v in self.data['tokens'].values():
 			self.conv_types(v)
 
-		if not is_coin_addr(g.token):
-			g.token = await self.sym2addr(g.token) # returns None on failure
-
-		if not is_coin_addr(g.token):
-			if self.importing:
-				m = 'When importing addresses for a new token, the token must be specified by address, not symbol.'
-				raise InvalidTokenAddress(f'{g.token!r}: invalid token address\n{m}')
-			else:
-				raise UnrecognizedTokenSymbol(f'Specified token {g.token!r} could not be resolved!')
-
-		if g.token in self.data['tokens']:
-			self.decimals = self.data['tokens'][g.token]['params']['decimals']
-			self.symbol = self.data['tokens'][g.token]['params']['symbol']
-		elif not self.importing:
-			raise TokenNotInWallet('Specified token {!r} not in wallet!'.format(g.token))
+		if self.importing and token_addr:
+			if not is_coin_addr(proto,token_addr):
+				raise InvalidTokenAddress(f'{token_addr!r}: invalid token address')
+		else:
+			assert token_addr == None,'EthereumTokenTrackingWallet_chk1'
+			token_addr = await self.sym2addr(proto.tokensym) # returns None on failure
+			if not is_coin_addr(proto,token_addr):
+				raise UnrecognizedTokenSymbol(f'Specified token {proto.tokensym!r} could not be resolved!')
+
+		from mmgen.obj import TokenAddr
+		self.token = TokenAddr(proto,token_addr)
+
+		if self.token in self.data['tokens']:
+			self.decimals = self.get_param('decimals')
+			self.symbol   = self.get_param('symbol')
+		elif self.importing:
+			await self.import_token(self.token) # sets self.decimals, self.symbol
+		else:
+			raise TokenNotInWallet(f'Specified token {self.token!r} not in wallet!')
 
-		self.token = g.token
-		g.proto.dcoin = self.symbol
+		proto.tokensym = self.symbol
 
 	async def is_in_wallet(self,addr):
 		return addr in self.data['tokens'][self.token]
@@ -189,7 +192,7 @@ def data_root_desc(self):
 		return 'token ' + self.get_param('symbol')
 
 	async def rpc_get_balance(self,addr):
-		return await Token(self.token,self.decimals).get_balance(addr)
+		return await Token(self.proto,self.token,self.decimals,self.rpc).get_balance(addr)
 
 	async def get_eth_balance(self,addr,force_rpc=False):
 		cache = self.cur_eth_balances
@@ -204,21 +207,19 @@ def get_param(self,param):
 		return self.data['tokens'][self.token]['params'][param]
 
 	@write_mode
-	async def import_token(tw):
+	async def import_token(self,tokenaddr):
 		"""
 		Token 'symbol' and 'decimals' values are resolved from the network by the system just
 		once, upon token import.  Thereafter, token address, symbol and decimals are resolved
 		either from the tracking wallet (online operations) or transaction file (when signing).
 		"""
-		if not g.token in tw.data['tokens']:
-			t = await TokenResolve(g.token)
-			tw.token = g.token
-			tw.data['tokens'][tw.token] = {
-				'params': {
-					'symbol': await t.get_symbol(),
-					'decimals': t.decimals
-				}
+		t = await TokenResolve(self.proto,self.rpc,tokenaddr)
+		self.data['tokens'][tokenaddr] = {
+			'params': {
+				'symbol': await t.get_symbol(),
+				'decimals': t.decimals
 			}
+		}
 
 # No unspent outputs with Ethereum, but naming must be consistent
 class EthereumTwUnspentOutputs(TwUnspentOutputs):
@@ -242,10 +243,10 @@ class EthereumTwUnspentOutputs(TwUnspentOutputs):
 		'q':'a_quit','p':'a_print','v':'a_view','w':'a_view_wide',
 		'l':'a_lbl_add','D':'a_addr_delete','R':'a_balance_refresh' }
 
-	async def __ainit__(self,*args,**kwargs):
+	async def __ainit__(self,proto,*args,**kwargs):
 		if g.use_cached_balances:
 			self.hdr_fmt += '\n' + yellow('WARNING: Using cached balances. These may be out of date!')
-		await TwUnspentOutputs.__ainit__(self,*args,**kwargs)
+		await TwUnspentOutputs.__ainit__(self,proto,*args,**kwargs)
 
 	def do_sort(self,key=None,reverse=False):
 		if key == 'txid': return
@@ -256,22 +257,15 @@ async def get_unspent_rpc(self):
 		if self.addrs:
 			wl = [d for d in wl if d['addr'] in self.addrs]
 		return [{
-				'account': TwLabel(d['mmid']+' '+d['comment'],on_fail='raise'),
+				'account': TwLabel(self.proto,d['mmid']+' '+d['comment'],on_fail='raise'),
 				'address': d['addr'],
 				'amount': await self.wallet.get_balance(d['addr']),
 				'confirmations': 0, # TODO
 				} for d in wl]
 
-	class MMGenTwUnspentOutput(MMGenListItem):
-		txid   = ListItemAttr('CoinTxID')
-		vout   = ListItemAttr(int,typeconv=False)
-		amt    = ImmutableAttr(lambda val:g.proto.coin_amt(val),typeconv=False)
-		amt2   = ListItemAttr(lambda val:g.proto.coin_amt(val),typeconv=False)
-		label  = ListItemAttr('TwComment',reassign_ok=True)
-		twmmid = ImmutableAttr('TwMMGenID')
-		addr   = ImmutableAttr('CoinAddr')
-		confs  = ImmutableAttr(int,typeconv=False)
-		skip   = ListItemAttr(str,typeconv=False,reassign_ok=True)
+	class MMGenTwUnspentOutput(TwUnspentOutputs.MMGenTwUnspentOutput):
+		valid_attrs = {'txid','vout','amt','amt2','label','twmmid','addr','confs','skip'}
+		invalid_attrs = {'proto'}
 
 	def age_disp(self,o,age_fmt): # TODO
 		return None
@@ -294,25 +288,26 @@ class EthereumTwAddrList(TwAddrList):
 
 	has_age = False
 
-	async def __ainit__(self,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
+	async def __ainit__(self,proto,usr_addr_list,minconf,showempty,showbtcaddrs,all_labels,wallet=None):
 
-		self.wallet = wallet or await TrackingWallet(mode='w')
+		self.proto = proto
+		self.wallet = wallet or await TrackingWallet(self.proto,mode='w')
 		tw_dict = self.wallet.mmid_ordered_dict
-		self.total = g.proto.coin_amt('0')
+		self.total = self.proto.coin_amt('0')
 
 		from mmgen.obj import CoinAddr
 		for mmid,d in list(tw_dict.items()):
 #			if d['confirmations'] < minconf: continue # cannot get confirmations for eth account
-			label = TwLabel(mmid+' '+d['comment'],on_fail='raise')
+			label = TwLabel(self.proto,mmid+' '+d['comment'],on_fail='raise')
 			if usr_addr_list and (label.mmid not in usr_addr_list):
 				continue
 			bal = await self.wallet.get_balance(d['addr'])
 			if bal == 0 and not showempty:
 				if not label.comment or not all_labels:
 					continue
-			self[label.mmid] = {'amt': g.proto.coin_amt('0'), 'lbl':  label }
+			self[label.mmid] = {'amt': self.proto.coin_amt('0'), 'lbl':  label }
 			if showbtcaddrs:
-				self[label.mmid]['addr'] = CoinAddr(d['addr'])
+				self[label.mmid]['addr'] = CoinAddr(self.proto,d['addr'])
 			self[label.mmid]['lbl'].mmid.confs = None
 			self[label.mmid]['amt'] += bal
 			self.total += bal
@@ -326,17 +321,17 @@ class EthereumTwGetBalance(TwGetBalance):
 
 	fs = '{w:13} {c}\n' # TODO - for now, just suppress display of meaningless data
 
-	async def __ainit__(self,*args,**kwargs):
-		self.wallet = await TrackingWallet(mode='w')
-		await TwGetBalance.__ainit__(self,*args,**kwargs)
+	async def __ainit__(self,proto,*args,**kwargs):
+		self.wallet = await TrackingWallet(proto,mode='w')
+		await TwGetBalance.__ainit__(self,proto,*args,**kwargs)
 
 	async def create_data(self):
 		data = self.wallet.mmid_ordered_dict
 		for d in data:
 			if d.type == 'mmgen':
 				key = d.obj.sid
 				if key not in self.data:
-					self.data[key] = [g.proto.coin_amt('0')] * 4
+					self.data[key] = [self.proto.coin_amt('0')] * 4
 			else:
 				key = 'Non-MMGen'
 
@@ -350,10 +345,9 @@ async def create_data(self):
 
 class EthereumTwAddrData(TwAddrData):
 
-	@classmethod
-	async def get_tw_data(cls,wallet=None):
+	async def get_tw_data(self,wallet=None):
 		vmsg('Getting address data from tracking wallet')
-		tw = (wallet or await TrackingWallet()).mmid_ordered_dict
+		tw = (wallet or await TrackingWallet(self.proto)).mmid_ordered_dict
 		# emulate the output of RPC 'listaccounts' and 'getaddressesbyaccount'
 		return [(mmid+' '+d['comment'],[d['addr']]) for mmid,d in list(tw.items())]