Merge branch 'threshold_py3_staging' into threshold

closes #9929

.gitmodules

@@ -21,8 +21,7 @@
 	ignore = untracked
 [submodule "include/comtypes"]
 	path = include/comtypes
-	url = https://github.com/enthought/comtypes.git
-	ignore = untracked
+	url = https://github.com/nvaccess/comtypes-bin
 [submodule "include/scons"]
 	path = include/scons
 	url = https://github.com/SConsProject/scons
@@ -38,3 +37,6 @@
 [submodule "include/cldr-emoji-annotation"]
 	path = include/cldr-emoji-annotation
 	url = https://github.com/fujiwarat/cldr-emoji-annotation
+[submodule "include/py2exe"]
+	path = include/py2exe
+	url = https://github.com/nvaccess/py2exe-bin

appveyor.yml

@@ -4,6 +4,7 @@ version: "{branch}-{build}"
 branches:
  only:
   - threshold
+  - threshold_py3_staging
   - master
   - beta
   - rc
@@ -12,7 +13,7 @@ branches:
   - /release-.*/
 
 environment:
- PY_PYTHON: 2.7-32
+ PY_PYTHON: 3.7-32
  encFileKey:
   secure: ekOvuyywHuDdGZmRmoj+b3jfrq39A2xlx4RD5ZUGd/8=
  mozillaSymsAuthToken:
@@ -150,7 +151,7 @@ test_script:
      $errorCode=0
      $outDir = (Resolve-Path .\testOutput\unit\)
      $unitTestsXml = "$outDir\unitTests.xml"
-     py -m nose --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
+     py -m nose -sv --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
      if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
      Push-AppveyorArtifact $unitTestsXml
      $wc = New-Object 'System.Net.WebClient'

appveyor/mozillaSyms.py

@@ -6,7 +6,6 @@
 To update the list of symbols uploaded to Mozilla, see the DLL_NAMES constant below.
 """
 
-from __future__ import print_function
 import argparse
 import os
 import subprocess
@@ -43,7 +42,8 @@ def __init__(self, returncode, stderr):
 def check_output(command):
 	proc = subprocess.Popen(command,
 		stdout=subprocess.PIPE,
-		stderr=subprocess.PIPE)
+		stderr=subprocess.PIPE,
+		text=True)
 	stdout, stderr = proc.communicate()
 	if proc.returncode != 0:
 		raise ProcError(proc.returncode, stderr)

appx/sconscript

@@ -31,7 +31,7 @@ def getCertPublisher(env):
 		return env['publisher']
 	certPassword=env.get('certPassword','')
 	cmd=['certutil','-dump','-p',certPassword,File('#'+certFile).abspath.replace('/','\\')]
-	lines=subprocess.check_output(cmd).splitlines()
+	lines=subprocess.run(cmd,check=True,capture_output=True,text=True).stdout.splitlines()
 	linePrefix='Subject: '
 	for line in lines:
 		if line.startswith(linePrefix):

cldrDict_sconscript

@@ -29,7 +29,7 @@ def createCLDRAnnotationsDict(sources, dest):
 	assert cldrDict, "cldrDict is empty"
 	with codecs.open(dest, "w", "utf_8_sig", errors="replace") as dictFile:
 		dictFile.write(u"symbols:\r\n")
-		for pattern, description in cldrDict.iteritems():
+		for pattern, description in cldrDict.items():
 			dictFile.write(u"{pattern}\t{description}\tsome\r\n".format(
 				pattern=pattern,
 				description=description
@@ -119,7 +119,7 @@ NVDAToCLDRLocales = {
 
 annotationsDir = env.Dir("include/cldr-emoji-annotation/annotations")
 annotationsDerivedDir = env.Dir("include/cldr-emoji-annotation/annotationsDerived")
-for destLocale, sourceLocales in NVDAToCLDRLocales.iteritems():
+for destLocale, sourceLocales in NVDAToCLDRLocales.items():
 	cldrSources = []
 	# First add all annotations, then the derived ones.
 	for sourceLocale in sourceLocales:

keyCommandsDoc.py

@@ -3,7 +3,7 @@
 #A part of NonVisual Desktop Access (NVDA)
 #This file is covered by the GNU General Public License.
 #See the file COPYING for more details.
-#Copyright (C) 2010-2015 NV Access Limited, Mesar Hameed
+#Copyright (C) 2010-2019 NV Access Limited, Mesar Hameed, Takuya Nishimoto
 
 """Utilities related to NVDA Key Commands documents.
 """
@@ -213,7 +213,7 @@ def _heading(self, m):
 		self._headings.append(m)
 		self._kcLastHeadingLevel = min(self._kcLastHeadingLevel, level - 1)
 
-	RE_SETTING_SINGLE_KEY = re.compile(ur"^[^|]+?[:：]\s*(.+?)\s*$")
+	RE_SETTING_SINGLE_KEY = re.compile(r"^[^|]+?[:：]\s*(.+?)\s*$")
 	def _handleSetting(self):
 		if not self._settingsHeaderRow:
 			raise KeyCommandsError("%d, setting command cannot be used before settingsSection command" % self._lineNum)
@@ -237,7 +237,7 @@ def _handleSetting(self):
 		# The next few lines should be table rows for each layout.
 		# Alternatively, if the key is common to all layouts, there will be a single line of text specifying the key after a colon.
 		keys = []
-		for layout in xrange(self._settingsNumLayouts):
+		for layout in range(self._settingsNumLayouts):
 			line = next(self._ug).strip()
 			self._lineNum += 1
 			m = self.RE_SETTING_SINGLE_KEY.match(line)
@@ -254,7 +254,7 @@ def _handleSetting(self):
 		if 1 == len(keys) < self._settingsNumLayouts:
 			# The key has only been specified once, so it is the same in all layouts.
 			key = keys[0]
-			keys[1:] = (key for layout in xrange(self._settingsNumLayouts - 1))
+			keys[1:] = (key for layout in range(self._settingsNumLayouts - 1))
 
 		# There should now be a blank line.
 		line = next(self._ug).strip()

nvdaHelper/archBuild_sconscript

@@ -34,7 +34,7 @@ def clsidStringToCLSIDDefine(clsidString):
 		"0x"+d[0:8],
 		"0x"+d[8:12],
 		"0x"+d[12:16],
-		"{%s}"%(",".join("0x"+d[x:x+2] for x in xrange(16,32,2)))
+		"{%s}"%(",".join("0x"+d[x:x+2] for x in range(16,32,2)))
 	)
 
 def COMProxyDllBuilder(env,target,source,proxyClsid):

nvdaHelper/espeak/sconscript

@@ -59,7 +59,7 @@ def espeak_compilePhonemeData_buildAction(target,source,env):
 	# Unfortunately, there's no way we can flush it or use a different stream
 	# because our eSpeak statically links the CRT.
 	espeak=AutoFreeCDLL(espeakLib[0].abspath)
-	espeak.espeak_ng_InitializePath(espeakRepo.abspath)
+	espeak.espeak_ng_InitializePath(os.fsencode(espeakRepo.abspath))
 	espeak.espeak_ng_CompileIntonation(None,None)
 	espeak.espeak_ng_CompilePhonemeData(22050,None,None)
 	espeak.espeak_Terminate()
@@ -72,14 +72,14 @@ def espeak_compileDict_buildAction(target,source,env):
 	# Unfortunately, there's no way we can flush it or use a different stream
 	# because our eSpeak statically links the CRT.
 	espeak=AutoFreeCDLL(espeakLib[0].abspath)
-	espeak.espeak_Initialize(0,0,target[0].Dir('..').abspath,0x8000)
+	espeak.espeak_Initialize(0,0,os.fsencode(target[0].Dir('..').abspath),0x8000)
 	try:
-		lang=source[0].name.split('_')[0]
-		v=espeak_VOICE(languages=lang+'\x00')
+		lang=source[0].name.split('_')[0].encode()
+		v=espeak_VOICE(languages=lang+b'\x00')
 		if espeak.espeak_SetVoiceByProperties(ctypes.byref(v))!=0:
 			print("espeak_compileDict_action: failed to switch to language %s"%lang)
 			return 1
-		dictPath=os.path.split(source[0].abspath)[0]+'/'
+		dictPath=os.fsencode(os.path.split(source[0].abspath)[0]+'/')
 		if espeak.espeak_ng_CompileDictionary(dictPath,None,0,None)!=0:
 			print("espeak_compileDict_action: failed to compile dictionary for  language %s"%lang)
 			return

nvdaHelper/liblouis/sconscript

@@ -1,7 +1,7 @@
 ###
 #This file is a part of the NVDA project.
 #URL: https://www.nvaccess.org/
-#Copyright 2011-2017 NV Access Limited, Joseph Lee
+#Copyright 2011-2018 NV Access Limited, Joseph Lee, Babbage B.V.
 #This program is free software: you can redistribute it and/or modify
 #it under the terms of the GNU General Public License version 2.0, as published by
 #the Free Software Foundation.
@@ -28,7 +28,7 @@ signExec=env['signExec'] if env['certFile'] else None
 RE_AC_INIT = re.compile(r"^AC_INIT\(\[(?P<package>.*)\], \[(?P<version>.*)\], \[(?P<bugReport>.*)\], \[(?P<tarName>.*)\], \[(?P<url>.*)\]\)")
 def getLouisVersion():
 	# Get the version from configure.ac.
-	with file(louisRootDir.File("configure.ac").abspath) as f:
+	with open(louisRootDir.File("configure.ac").abspath) as f:
 		for line in f:
 			m = RE_AC_INIT.match(line)
 			if m:
@@ -54,7 +54,7 @@ env.Append(CPPDEFINES=[
 	# variants that start with an '_'. This removes those deprecation warnings. */
 	"_CRT_NONSTDC_NO_DEPRECATE",
 	("PACKAGE_VERSION", r'\"%s\"' % getLouisVersion()),
-	("UNICODE_BITS", 16),
+	"WIDECHARS_ARE_UCS4",
 	# Tell liblouis.h that we're exporting liblouis dll functions, not importing them.
 	"_EXPORTING",
 ])
@@ -64,7 +64,7 @@ env.Prepend(CPPPATH=[".", louisSourceDir])
 env['CPPDEFINES'].remove("UNICODE")
 
 liblouisH = env.Substfile("liblouis.h", louisSourceDir.File("liblouis.h.in"),
-	SUBST_DICT={"@WIDECHAR_TYPE@": "unsigned short int"})
+	SUBST_DICT={"@WIDECHAR_TYPE@": "unsigned int"})
 
 sourceFiles = [
 	"compileTranslationTable.c",

readme.md

@@ -57,31 +57,32 @@ If you didn't pass the `--recursive` option to git clone, you will need to run `
 Whenever a required submodule commit changes (e.g. after git pull), you will need to run `git submodule update`.
 If you aren't sure, run `git submodule update` after every git pull, merge or checkout.
 
-For reference, the following dependencies are included in Git submodules:
+For reference, the following run time dependencies are included in Git submodules:
 
 * [comtypes](https://github.com/enthought/comtypes), version 1.1.7
 * [wxPython](http://www.wxpython.org/), version 4.0.3
-* [Six](https://pypi.python.org/pypi/six), version 1.10.0, required by wxPython
-* [Python Windows Extensions](http://sourceforge.net/projects/pywin32/ ), build 218
 * [eSpeak NG](https://github.com/espeak-ng/espeak-ng), commit 86e67a
 * [Sonic](https://github.com/waywardgeek/sonic), commit 4f8c1d11
 * [IAccessible2](http://www.linuxfoundation.org/collaborate/workgroups/accessibility/iaccessible2), commit 21bbb176
 * [ConfigObj](https://github.com/DiffSK/configobj), commit 5b5de48
+* [Six](https://pypi.python.org/pypi/six), version 1.12.0, required by wxPython and ConfigObj
 * [liblouis](http://www.liblouis.org/), version 3.10.0
 * [Unicode Common Locale Data Repository (CLDR)](http://cldr.unicode.org/) Emoji Annotations, version 35.0
 * NVDA images and sounds
-* System dlls not present on many systems: mfc90.dll, msvcp90.dll, msvcr90.dll, Microsoft.VC90.CRT.manifest
 * [Adobe Acrobat accessibility interface, version XI](http://download.macromedia.com/pub/developer/acrobat/AcrobatAccess.zip)
 * Adobe FlashAccessibility interface typelib
-* [txt2tags](http://txt2tags.sourceforge.net/), version 2.5
 * [MinHook](https://github.com/RaMMicHaeL/minhook), tagged version 1.2.2
-* [SCons](http://www.scons.org/), version 3.0.4
-* brlapi Python bindings, version 0.5.7 or later, distributed with [BRLTTY for Windows](http://brl.thefreecat.org/brltty/), version 4.2-2
-* ALVA BC6 generic dll, version 3.0.4.1
+* brlapi Python bindings, version 0.7.0 or later, distributed with [BRLTTY for Windows](http://brl.thefreecat.org/brltty/), version 4.2-2
 * lilli.dll, version 2.1.0.0
 * [pySerial](http://pypi.python.org/pypi/pyserial), version 3.4
 * [Python interface to FTDI driver/chip](http://fluidmotion.dyndns.org/zenphoto/index.php?p=news&title=Python-interface-to-FTDI-driver-chip)
-* [Py2Exe](http://sourceforge.net/projects/py2exe/), version 0.6.9
+
+Additionally, the following build time dependencies are included in Git submodules:
+
+* [Py2Exe](http://github.com/albertosottile/py2exe/), version 0.9.3.2 commit b372a8e
+* [Python Windows Extensions](http://sourceforge.net/projects/pywin32/ ), build 224, required by py2exe
+* [txt2tags](http://txt2tags.sourceforge.net/), version 2.5
+* [SCons](http://www.scons.org/), version 3.0.4
 * [Nulsoft Install System](http://nsis.sourceforge.net/), version 2.51
 * [NSIS UAC plug-in](http://nsis.sourceforge.net/UAC_plug-in), version 0.2.4, ansi
 * xgettext and msgfmt from [GNU gettext](http://sourceforge.net/projects/cppcms/files/boost_locale/gettext_for_windows/)
@@ -91,7 +92,7 @@ For reference, the following dependencies are included in Git submodules:
 ### Other Dependencies
 These dependencies are not included in Git submodules, but aren't needed by most people.
 
-* To generate developer documentation for nvdaHelper: [Doxygen version 1.7.3 Windows installer](https://sourceforge.net/projects/doxygen/files/rel-1.7.3/doxygen-1.7.3-setup.exe)
+* To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.doxygen.nl/download.html), version 1.8.15:
 
 ## Preparing the Source Tree
 Before you can run the NVDA source code, you must prepare the source tree.
@@ -161,13 +162,14 @@ scons launcher
 
 The archive will be placed in the output directory.
 
-To generate developer documentation, type:
+To generate the NVDA developer guide, type:
 
 ```
-scons devDocs
+scons developerGuide
 ```
 
-The developer docs will be placed in the `devDocs` folder in the output directory.
+The developer guide will be placed in the `devDocs` folder in the output directory.
+Note that the Python 3 sources of NVDA currently do not support building NVDA developer documentation using the `scons devDocs` command.
 
 To generate developer documentation for nvdaHelper (not included in the devDocs target):
 

scons.bat

@@ -5,8 +5,8 @@ rem Instead, find the python launcher (installed by python 3)
 where py 1>nul 2>&1
 if "%ERRORLEVEL%" == "0" (
 	rem Python launcher is present in the PATH
-	rem Call python 2.7 for 32 bits
-	py -2.7-32 "%~dp0\scons.py" %*
+	rem Call python 3.7 for 32 bits
+	py -3.7-32 "%~dp0\scons.py" %*
 ) else (
 	rem Python registers itself with the .py extension, so call scons.py.
 	"%~dp0\scons.py" %*

scons.py

@@ -5,7 +5,7 @@
 import os
 import platform
 # Variables for storing required version of Python, and the version which is used to run this script.
-requiredPythonMajor ="2"
+requiredPythonMajor ="3"
 requiredPythonMinor = "7"
 requiredPythonArchitecture = "32bit"
 installedPythonMajor = str(sys.version_info.major)