Support MSWin via MSYS2

This is a work in progress.  Currently, basic operations for BTC and ETH are
supported.

The successor to the MinGW64 project, MSYS2 features package management via
`pacman` and support for Python 3:

    https://sourceforge.net/projects/msys2
    https://www.msys2.org

mmgen/globalvars.py

@@ -102,6 +102,7 @@ def die(ev=0,s=''):
 	debug_utf8           = False
 	traceback            = False
 	test_suite           = False
+	test_suite_popen_spawn = False
 
 	for k in ('linux','win','msys'):
 		if sys.platform[:len(k)] == k:
@@ -166,6 +167,7 @@ def die(ev=0,s=''):
 		'MMGEN_DEBUG_ALL', # special: there is no g.debug_all var
 
 		'MMGEN_TEST_SUITE',
+		'MMGEN_TEST_SUITE_POPEN_SPAWN',
 		'MMGEN_BOGUS_WALLET_DATA',
 		'MMGEN_BOGUS_SEND',
 		'MMGEN_DEBUG',

mmgen/main.py

@@ -28,8 +28,13 @@ def launch(what):
 
 	import sys
 
-	try: import termios
-	except: # Windows
+	try:
+		import termios
+		platform = 'linux'
+	except:
+		platform = 'win'
+
+	if platform == 'win':
 		__import__('mmgen.main_' + what)
 	else:
 		import os,atexit

mmgen/regtest.py

@@ -235,14 +235,22 @@ def setup():
 
 def get_current_user_win(quiet=False):
 	if test_daemon() == 'stopped': return None
-	p = start_cmd('grep','Using wallet',os.path.join(daemon_dir,'debug.log'),quiet=True)
-	try: wallet_fn = p.stdout.readlines()[-1].split()[-1].decode()
-	except: return None
-	for k in ('miner','bob','alice'):
-		if wallet_fn == 'wallet.dat.'+k:
-			if not quiet: msg('Current user is {}'.format(k.capitalize()))
-			return k
-	return None
+	logfile = os.path.join(daemon_dir,'debug.log')
+	p = start_cmd('grep','Wallet completed loading in',logfile,quiet=True)
+	last_line = p.stdout.readlines()[-1].decode()
+
+	import re
+	m = re.search(r'\[wallet.dat.([a-z]+)\]',last_line)
+	if not m:
+		return None
+
+	user = m.group(1)
+	if user in ('miner','bob','alice'):
+		if not quiet:
+			msg('Current user is {}'.format(user.capitalize()))
+		return user
+	else:
+		return None
 
 def get_current_user_unix(quiet=False):
 	p = start_cmd('pgrep','-af','{}.*--rpcport={}.*'.format(g.proto.daemon_name,rpc_port),quiet=True)
@@ -254,7 +262,7 @@ def get_current_user_unix(quiet=False):
 			return k
 	return None
 
-get_current_user = (get_current_user_win,get_current_user_unix)[g.platform=='linux']
+get_current_user = { 'win':get_current_user_win, 'linux':get_current_user_unix }[g.platform]
 
 def bob():   return user('bob',quiet=False)
 def alice(): return user('alice',quiet=False)

mmgen/term.py

@@ -133,12 +133,12 @@ def _get_keypress_mswin(prompt='',immed_chars='',prehold_protect=True,num_chars=
 def _get_keypress_mswin_raw(prompt='',immed_chars='',prehold_protect=None,num_chars=None):
 	msg_r(prompt)
 	ch = msvcrt.getch()
-	if ord(ch) == 3: raise KeyboardInterrupt
+	if ch == 3: raise KeyboardInterrupt
 	return ch
 
 def _get_keypress_mswin_stub(prompt='',immed_chars='',prehold_protect=None,num_chars=None):
 	msg_r(prompt)
-	return sys.stdin.read(1)
+	return os.read(0,1)
 
 def _get_terminal_size_linux():
 	try:

mmgen/util.py

@@ -634,9 +634,12 @@ def do_file(outfile,ask_write_prompt):
 					m = "{} in file '{}' has been altered by some other program!  Aborting file write"
 					die(3,m.format(desc,outfile))
 
-		f = open_file_or_exit(outfile,('w','wb')[bool(binary)])
+		# To maintain portability, always open files in binary mode
+		# If 'binary' option not set, encode/decode data before writing and after reading
+		f = open_file_or_exit(outfile,'wb')
+
 		try:
-			f.write(data)
+			f.write(data if binary else data.encode())
 		except:
 			die(2,"Failed to write {} to file '{}'".format(desc,outfile))
 		f.close
@@ -654,16 +657,15 @@ def do_file(outfile,ask_write_prompt):
 		do_file(outfile,ask_write_prompt)
 
 def get_words_from_user(prompt):
-	# split() also strips
 	words = my_raw_input(prompt, echo=opt.echo_passphrase).split()
 	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
 	return words
 
 def get_words_from_file(infile,desc,quiet=False):
 	if not quiet:
 		qmsg("Getting {} from file '{}'".format(desc,infile))
-	f = open_file_or_exit(infile, 'r')
-	try: words = f.read().split() # split() also strips
+	f = open_file_or_exit(infile, 'rb')
+	try: words = f.read().decode().split()
 	except: die(1,'{} data must be UTF-8 encoded.'.format(capfirst(desc)))
 	f.close()
 	dmsg('Sanitized input: [{}]'.format(' '.join(words)))
@@ -687,7 +689,7 @@ def mmgen_decrypt_file_maybe(fn,desc='',quiet=False,silent=False):
 
 def get_lines_from_file(fn,desc='',trim_comments=False,quiet=False,silent=False):
 	dec = mmgen_decrypt_file_maybe(fn,desc,quiet=quiet,silent=silent)
-	ret = dec.decode('utf8').splitlines() # DOS-safe
+	ret = dec.decode().splitlines()
 	if trim_comments: ret = remove_comments(ret)
 	dmsg("Got {} lines from file '{}'".format(len(ret),fn))
 	return ret
@@ -703,12 +705,13 @@ def get_data_from_file(infile,desc='data',dash=False,silent=False,binary=False,q
 	if not opt.quiet and not silent and not quiet and desc:
 		qmsg("Getting {} from file '{}'".format(desc,infile))
 
-	mode = ('r','rb')[bool(binary)]
-
 	if dash and infile == '-':
-		data = os.fdopen(0,mode).read(g.max_input_size+1)
+		data = os.fdopen(0,'rb').read(g.max_input_size+1)
 	else:
-		data = open_file_or_exit(infile,mode,silent=silent).read(g.max_input_size+1)
+		data = open_file_or_exit(infile,'rb',silent=silent).read(g.max_input_size+1)
+
+	if not binary:
+		data = data.decode()
 
 	if len(data) == g.max_input_size + 1:
 		raise MaxInputSizeExceeded('Too much input data!  Max input data size: {} bytes'.format(g.max_input_size))
@@ -744,11 +747,16 @@ def st_hook(): readline.insert_text(insert_txt)
 
 	from mmgen.term import kb_hold_protect
 	kb_hold_protect()
-	if echo or not sys.stdin.isatty():
+
+	if g.test_suite_popen_spawn:
+		msg(prompt)
+		reply = os.read(0,4096).decode()
+	elif echo or not sys.stdin.isatty():
 		reply = input(prompt)
 	else:
 		from getpass import getpass
 		reply = getpass(prompt)
+
 	kb_hold_protect()
 
 	try:

scripts/traceback_run.py

@@ -4,7 +4,7 @@
 # file, as all names will be seen by the exec'ed file.  To prevent name collisions, all names
 # defined here should begin with 'traceback_run_'
 
-import sys,time
+import sys,os,time
 
 def traceback_run_init():
 	import os
@@ -13,7 +13,7 @@ def traceback_run_init():
 	if 'TMUX' in os.environ: del os.environ['TMUX']
 	os.environ['MMGEN_TRACEBACK'] = '1'
 
-	of = os.path.join(os.environ['PWD'],'my.err')
+	of = 'my.err'
 	try: os.unlink(of)
 	except: pass
 
@@ -28,10 +28,12 @@ def traceback_run_process_exception():
 
 	exc = l.pop()
 	if exc[:11] == 'SystemExit:': l.pop()
-
-	red    = lambda s: '\033[31;1m{}\033[0m'.format(s)
-	yellow = lambda s: '\033[33;1m{}\033[0m'.format(s)
-	sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc)))
+	if os.getenv('MMGEN_DISABLE_COLOR'):
+		sys.stdout.write('{}{}'.format(''.join(l),exc))
+	else:
+		red    = lambda s: '\033[31;1m{}\033[0m'.format(s)
+		yellow = lambda s: '\033[33;1m{}\033[0m'.format(s)
+		sys.stdout.write('{}{}'.format(yellow(''.join(l)),red(exc)))
 
 	open(traceback_run_outfile,'w').write(''.join(l+[exc]))
 
@@ -50,10 +52,5 @@ def traceback_run_process_exception():
 	traceback_run_process_exception()
 	sys.exit(e.mmcode if hasattr(e,'mmcode') else e.code if hasattr(e,'code') else 1)
 
-blue = lambda s: '\033[34;1m{}\033[0m'.format(s)
+blue = lambda s: s if os.getenv('MMGEN_DISABLE_COLOR') else '\033[34;1m{}\033[0m'.format(s)
 sys.stdout.write(blue('Runtime: {:0.5f} secs\n'.format(time.time() - traceback_run_tstart)))
-
-# else:
-# 	print('else: '+repr(sys.exc_info()))
-# finally:
-# 	print('finally: '+repr(sys.exc_info()))

setup.py

@@ -26,43 +26,8 @@
 	sys.stderr.write(m.format(*ver,M=min_ver[0],m=min_ver[1]))
 	sys.exit(1)
 
-_gvi = subprocess.check_output(['gcc','--version']).decode().splitlines()[0]
-have_mingw64 = 'x86_64' in _gvi and 'MinGW' in _gvi
-have_arm     = subprocess.check_output(['uname','-m']).strip() == 'aarch64'
-have_msys2   = not have_mingw64 and os.getenv('MSYSTEM') == 'MINGW64'
-
-# Zipfile module under Windows (MinGW) can't handle UTF-8 filenames.
-# Move it so that distutils will use the 'zip' utility instead.
-def divert_zipfile_module():
-	msg1 = 'Unable to divert zipfile module. UTF-8 filenames may be broken in the Python archive.'
-	def return_warn(m):
-		sys.stderr.write('WARNING: {}\n'.format(m))
-		return False
-
-	dirname = os.path.dirname(sys.modules['os'].__file__)
-	if not dirname: return return_warn(msg1)
-	stem = os.path.join(dirname,'zipfile')
-	a,b = stem+'.py',stem+'-is-broken.py'
-
-	try: os.stat(a)
-	except: return
-
-	try:
-		sys.stderr.write('moving {} -> {}\n'.format(a,b))
-		os.rename(a,b)
-	except:
-		return return_warn(msg1)
-	else:
-		try:
-			os.unlink(stem+'.pyc')
-			os.unlink(stem+'.pyo')
-		except:
-			pass
-
-if have_mingw64:
-# 	import zipfile
-# 	sys.exit()
-	divert_zipfile_module()
+have_arm   = subprocess.check_output(['uname','-m']).strip() == b'aarch64'
+have_msys2 = subprocess.check_output(['uname','-s']).strip()[:7] == b'MSYS_NT'
 
 from distutils.core import setup,Extension
 from distutils.command.build_ext import build_ext
@@ -92,7 +57,6 @@ def run(self):
 	libraries    = ['secp256k1'],
 	library_dirs = ['/usr/local/lib',r'c:\msys\local\lib'],
 	# mingw32 needs this, Linux can use it, but it breaks mingw64
-	extra_link_args = (['-lgmp'],[])[have_mingw64],
 	include_dirs = ['/usr/local/include',r'c:\msys\local\include'],
 	)
 

test/pexpect.py

@@ -37,12 +37,7 @@ def debug_pexpect_msg(p):
 		msg('\n{}{}{}'.format(red('BEFORE ['),p.before,red(']')))
 		msg('{}{}{}'.format(red('MATCH ['),p.after,red(']')))
 
-if g.platform == 'linux' and not opt.pexpect_spawn:
-	import atexit
-	atexit.register(lambda: os.system('stty sane'))
-	NL = '\n'
-else:
-	NL = '\r\n'
+NL = '\n'
 
 class MMGenPexpect(object):
 

test/test.py

@@ -42,6 +42,7 @@ def create_shm_dir(data_dir,trash_dir):
 						time.sleep(2)
 						shutil.rmtree(tdir)
 			os.mkdir(tdir,0o755)
+		shm_dir = 'test'
 	else:
 		tdir,pfx = '/dev/shm','mmgen-test-'
 		try:
@@ -566,6 +567,9 @@ def spawn_wrapper(  self, cmd,
 
 		args = [cmd] + passthru_opts + ['--data-dir='+self.data_dir] + args
 
+		if opt.traceback:
+			args = ['scripts/traceback_run.py'] + args
+
 		if g.platform == 'win':
 			args = ['python3'] + args
 
@@ -576,8 +580,6 @@ def spawn_wrapper(  self, cmd,
 
 		if opt.coverage:
 			args = ['python3','-m','trace','--count','--coverdir='+self.coverdir,'--file='+self.accfile] + args
-		elif opt.traceback:
-			args = ['scripts/traceback_run.py'] + args
 
 		qargs = ['{q}{}{q}'.format(a,q=('',"'")[' ' in a]) for a in args]
 		cmd_disp = ' '.join(qargs).replace('\\','/') # for mingw
@@ -594,7 +596,11 @@ def spawn_wrapper(  self, cmd,
 
 		if msg_only: return
 
-		if opt.log: self.log_fd.write(cmd_disp+'\n')
+		if opt.log:
+			try:
+				self.log_fd.write(cmd_disp+'\n')
+			except:
+				self.log_fd.write(ascii(cmd_disp)+'\n')
 
 		from test.pexpect import MMGenPexpect
 		return MMGenPexpect(args,no_output=no_output)

test/test_py_d/common.py

@@ -93,10 +93,13 @@ def iqmsg(s):
 def iqmsg_r(s):
 	if not opt.quiet: omsg_r(s)
 
-devnull_fh = open('/dev/null','w')
-def silence():
-	if not (opt.verbose or opt.exact_output):
-		g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno()
+if g.platform == 'win':
+	def silence(): pass
+else:
+	devnull_fh = open('/dev/null','w')
+	def silence():
+		if not (opt.verbose or opt.exact_output):
+			g.stderr_fileno = g.stdout_fileno = devnull_fh.fileno()
 
 def end_silence():
 	if not (opt.verbose or opt.exact_output):