Update Scons build environment to run on Python 3 (#9667)

* Update batch script and python wrapper for scons

* * Update copyright
* Remove obsolete _winreg import
* No longer pass unicode=True to gettext.install

* * sconstruct: xrange>range
* comInterfaces_sconscript: iteritems>items
* comInterfaces_sconscript: basestring>str
* nvdaHelper/archBuild_sconscript: xrange>range
* Liblouis sconscript: file>open
* cldrDict_sconscript: iteritems>items

* Disable developer documentation for now

* comInterfaces_sconscript: make sure that the byte compiled targets are picked up correctly by scons

* Fix espeak initialization and paths

* sconstruct: file>open

* Convert doxygen script to Python 3 and fix doxygen site tool

* Fix pot files creation

* Fix several issues to the dist builder

* Issue errors about str(bytes_instance), str(bytearray_instance)

* Update readme

cldrDict_sconscript

@@ -29,7 +29,7 @@ def createCLDRAnnotationsDict(sources, dest):
 	assert cldrDict, "cldrDict is empty"
 	with codecs.open(dest, "w", "utf_8_sig", errors="replace") as dictFile:
 		dictFile.write(u"symbols:\r\n")
-		for pattern, description in cldrDict.iteritems():
+		for pattern, description in cldrDict.items():
 			dictFile.write(u"{pattern}\t{description}\tsome\r\n".format(
 				pattern=pattern,
 				description=description
@@ -119,7 +119,7 @@ NVDAToCLDRLocales = {
 
 annotationsDir = env.Dir("include/cldr-emoji-annotation/annotations")
 annotationsDerivedDir = env.Dir("include/cldr-emoji-annotation/annotationsDerived")
-for destLocale, sourceLocales in NVDAToCLDRLocales.iteritems():
+for destLocale, sourceLocales in NVDAToCLDRLocales.items():
 	cldrSources = []
 	# First add all annotations, then the derived ones.
 	for sourceLocale in sourceLocales:

nvdaHelper/archBuild_sconscript

@@ -34,7 +34,7 @@ def clsidStringToCLSIDDefine(clsidString):
 		"0x"+d[0:8],
 		"0x"+d[8:12],
 		"0x"+d[12:16],
-		"{%s}"%(",".join("0x"+d[x:x+2] for x in xrange(16,32,2)))
+		"{%s}"%(",".join("0x"+d[x:x+2] for x in range(16,32,2)))
 	)
 
 def COMProxyDllBuilder(env,target,source,proxyClsid):

nvdaHelper/espeak/sconscript

@@ -59,7 +59,7 @@ def espeak_compilePhonemeData_buildAction(target,source,env):
 	# Unfortunately, there's no way we can flush it or use a different stream
 	# because our eSpeak statically links the CRT.
 	espeak=AutoFreeCDLL(espeakLib[0].abspath)
-	espeak.espeak_ng_InitializePath(espeakRepo.abspath)
+	espeak.espeak_ng_InitializePath(os.fsencode(espeakRepo.abspath))
 	espeak.espeak_ng_CompileIntonation(None,None)
 	espeak.espeak_ng_CompilePhonemeData(22050,None,None)
 	espeak.espeak_Terminate()
@@ -72,14 +72,14 @@ def espeak_compileDict_buildAction(target,source,env):
 	# Unfortunately, there's no way we can flush it or use a different stream
 	# because our eSpeak statically links the CRT.
 	espeak=AutoFreeCDLL(espeakLib[0].abspath)
-	espeak.espeak_Initialize(0,0,target[0].Dir('..').abspath,0x8000)
+	espeak.espeak_Initialize(0,0,os.fsencode(target[0].Dir('..').abspath),0x8000)
 	try:
-		lang=source[0].name.split('_')[0]
-		v=espeak_VOICE(languages=lang+'\x00')
+		lang=source[0].name.split('_')[0].encode()
+		v=espeak_VOICE(languages=lang+b'\x00')
 		if espeak.espeak_SetVoiceByProperties(ctypes.byref(v))!=0:
 			print("espeak_compileDict_action: failed to switch to language %s"%lang)
 			return 1
-		dictPath=os.path.split(source[0].abspath)[0]+'/'
+		dictPath=os.fsencode(os.path.split(source[0].abspath)[0]+'/')
 		if espeak.espeak_ng_CompileDictionary(dictPath,None,0,None)!=0:
 			print("espeak_compileDict_action: failed to compile dictionary for  language %s"%lang)
 			return

nvdaHelper/liblouis/sconscript

@@ -28,7 +28,7 @@ signExec=env['signExec'] if env['certFile'] else None
 RE_AC_INIT = re.compile(r"^AC_INIT\(\[(?P<package>.*)\], \[(?P<version>.*)\], \[(?P<bugReport>.*)\], \[(?P<tarName>.*)\], \[(?P<url>.*)\]\)")
 def getLouisVersion():
 	# Get the version from configure.ac.
-	with file(louisRootDir.File("configure.ac").abspath) as f:
+	with open(louisRootDir.File("configure.ac").abspath) as f:
 		for line in f:
 			m = RE_AC_INIT.match(line)
 			if m:

readme.md

@@ -91,7 +91,7 @@ For reference, the following dependencies are included in Git submodules:
 ### Other Dependencies
 These dependencies are not included in Git submodules, but aren't needed by most people.
 
-* To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.stack.nl/~dimitri/doxygen/download.html), version 1.7.3:
+* To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.doxygen.nl/download.html), version 1.8.15:
 
 ## Preparing the Source Tree
 Before you can run the NVDA source code, you must prepare the source tree.
@@ -161,13 +161,14 @@ scons launcher
 
 The archive will be placed in the output directory.
 
-To generate developer documentation, type:
+To generate the NVDA developer guide, type:
 
 ```
-scons devDocs
+scons developerGuide
 ```
 
-The developer docs will be placed in the `devDocs` folder in the output directory.
+The developer guide will be placed in the `devDocs` folder in the output directory.
+Note that the Python 3 sources of NVDA currently do not support building NVDA developer documentation using the `scons devDocs` command.
 
 To generate developer documentation for nvdaHelper (not included in the devDocs target):
 

scons.bat

@@ -5,8 +5,8 @@ rem Instead, find the python launcher (installed by python 3)
 where py 1>nul 2>&1
 if "%ERRORLEVEL%" == "0" (
 	rem Python launcher is present in the PATH
-	rem Call python 2.7 for 32 bits
-	py -2.7-32 "%~dp0\scons.py" %*
+	rem Call python 3.7 for 32 bits
+	py -3.7-32 "%~dp0\scons.py" %*
 ) else (
 	rem Python registers itself with the .py extension, so call scons.py.
 	"%~dp0\scons.py" %*

scons.py

@@ -5,7 +5,7 @@
 import os
 import platform
 # Variables for storing required version of Python, and the version which is used to run this script.
-requiredPythonMajor ="2"
+requiredPythonMajor ="3"
 requiredPythonMinor = "7"
 requiredPythonArchitecture = "32bit"
 installedPythonMajor = str(sys.version_info.major)

sconstruct

@@ -1,7 +1,7 @@
 ###
 #This file is a part of the NVDA project.
 #URL: https://www.nvaccess.org/
-#Copyright 2010-2017 NV Access Limited.
+#Copyright 2010-2019 NV Access Limited, Babbage B.V.
 #This program is free software: you can redistribute it and/or modify
 #it under the terms of the GNU General Public License version 2.0, as published by
 #the Free Software Foundation.
@@ -15,9 +15,9 @@
 import sys
 import os
 import time
-import _winreg
 from glob import glob
 import sourceEnv
+import importlib.util
 
 def recursiveCopy(env,targetDir,sourceDir):
 	targets=[]
@@ -35,7 +35,7 @@ def recursiveCopy(env,targetDir,sourceDir):
 
 # Import NVDA's versionInfo module.
 import gettext
-gettext.install("nvda", unicode=True)
+gettext.install("nvda")
 sys.path.append("source")
 import versionInfo
 del sys.path[-1]
@@ -72,14 +72,21 @@ vars.Add("certPassword", "The password for the private key in the signing certif
 vars.Add("certTimestampServer", "The URL of the timestamping server to use to timestamp authenticode signatures", "")
 vars.Add(PathVariable("outputDir", "The directory where the final built archives and such will be placed", "output",PathVariable.PathIsDirCreate))
 vars.Add(ListVariable("nvdaHelperDebugFlags", "a list of debugging features you require", 'none', ["debugCRT","RTC","analyze"]))
-vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in xrange(60)]))
+vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in range(60)]))
 if "tests" in COMMAND_LINE_TARGETS:
 	vars.Add("unitTests", "A list of unit tests to run", "")
 if "systemTests" in COMMAND_LINE_TARGETS:
 	vars.Add("filter", "A filter for the name of the system test(s) to run. Wildcards accepted.", "")
 
 #Base environment for this and sub sconscripts
-env = Environment(variables=vars,HOST_ARCH='x86',tools=["textfile","gettextTool","t2t",keyCommandsDocTool,'doxygen','recursiveInstall'])
+env = Environment(variables=vars,HOST_ARCH='x86',tools=[
+	"textfile",
+	"gettextTool",
+	"t2t"
+	,keyCommandsDocTool,
+	"doxygen",
+	"recursiveInstall"
+])
 
 # speed up subsiquent runs by checking timestamps of targets and dependencies, and only using md5 if timestamps differ.
 env.Decider('MD5-timestamp')
@@ -139,7 +146,7 @@ def signExec(target,source,env):
 	#sys.exit(1)
 	# #3795: signtool can quite commonly fail with timestamping, so allow it to try up to 3 times with a 1 second delay between each try. 
 	res=0
-	for count in xrange(3):
+	for count in range(3):
 		res=env.Execute([signExecCmd+[target[0].abspath]])
 		if not res:
 			return 0 # success
@@ -228,31 +235,41 @@ def NVDADistGenerator(target, source, env, for_signature):
 	# We don't do this using normal scons mechanisms because we want it to be cleaned up immediately after this builder
 	# and py2exe will cause bytecode files to be created for it which scons doesn't know about.
 	updateVersionType = env["updateVersionType"] or None
-	action = [lambda target, source, env: file(buildVersionFn, "w").write(
-		'version = {version!r}\r\n'
-		'publisher = {publisher!r}\r\n'
-		'updateVersionType = {updateVersionType!r}\r\n'
-		'version_build = {version_build!r}\r\n'
-		.format(version=version, publisher=publisher, updateVersionType=updateVersionType,version_build=version_build))]
+	# Any '\n' characters written are translated to the system default line separator, os.linesep.
+	action = [lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write(
+		'version = {version!r}\n'
+		'publisher = {publisher!r}\n'
+		'updateVersionType = {updateVersionType!r}\n'
+		'version_build = {version_build!r}\n'
+		.format(version=version, publisher=publisher, updateVersionType=updateVersionType,version_build=version_build)
+	)
+	# In Python 3 write returns the number of characters written,
+	# which scons treats as an error code.	
+	and None]
 
 	buildCmd = ["cd", source[0].path, "&&",
 		sys.executable]
 	if release:
 		buildCmd.append("-O")
+	# Issue errors about str(bytes_instance), str(bytearray_instance)
+	buildCmd.append("-bb")
 	buildCmd.extend(("setup.py", "build", "--build-base", buildDir.abspath,
 		"py2exe", "--dist-dir", target[0].abspath))
 	if release:
 		buildCmd.append("-O1")
-	action.append(buildCmd)
-
 	if env.get("uiAccess"):
 		buildCmd.append("--enable-uiAccess")
+
+	action.append(buildCmd)
+
 	if certFile:
 		for prog in "nvda_noUIAccess.exe", "nvda_uiAccess.exe", "nvda_slave.exe", "nvda_eoaProxy.exe":
 			action.append(lambda target,source,env, progByVal=prog: signExec([target[0].File(progByVal)],source,env))
 
-	for ext in "", "c", "o":
-		action.append(Delete(buildVersionFn + ext))
+	action.extend((
+		Delete(buildVersionFn),
+		Delete(importlib.util.cache_from_source(buildVersionFn))
+	))
 
 	return action
 env["BUILDERS"]["NVDADist"] = Builder(generator=NVDADistGenerator, target_factory=Dir)
@@ -353,7 +370,7 @@ def makePot(target, source, env):
 	# Tweak the headers.
 	potFn = str(target[0])
 	tmpFn = "%s.tmp" % potFn
-	with file(potFn, "rt") as inp, file(tmpFn, "wt") as out:
+	with open(potFn, "rt") as inp, open(tmpFn, "wt") as out:
 		for lineNum, line in enumerate(inp):
 			if lineNum == 1:
 				line = "# %s\n" % versionInfo.copyright
@@ -371,19 +388,19 @@ devDocs_nvdaHelper=env.Command(devDocsOutputDir.Dir('nvdaHelper'),devDocs_nvdaHe
 env.Alias('devDocs_nvdaHelper', devDocs_nvdaHelper)
 env.Clean('devDocs_nvdaHelper', devDocs_nvdaHelper)
 
-devDocs_nvda = env.Command(devDocsOutputDir.Dir("nvda"), None, [[
-	"cd", sourceDir.path, "&&",
-	sys.executable, "-c", "import sourceEnv; from epydoc.cli import cli; cli()",
-	"--output", "${TARGET.abspath}",
-	"--quiet", "--html", "--include-log", "--no-frames",
-	"--name", "NVDA", "--url", "https://www.nvaccess.org/",
-	"*.py", "appModules", "brailleDisplayDrivers", r"comInterfaces\__init__.py",
-	"config", "contentRecog", "extensionPoints", "globalPlugins", "gui", "mathPres", "NVDAObjects",
-	"speechDictHandler", "synthDrivers", "textInfos", "virtualBuffers",
-]])
-
-env.Alias('devDocs', [devGuide, devDocs_nvda])
-env.Clean('devDocs', [devGuide, devDocs_nvda])
+#devDocs_nvda = env.Command(devDocsOutputDir.Dir("nvda"), None, [[
+#	"cd", sourceDir.path, "&&",
+#	sys.executable, "-c", "import sourceEnv; from epydoc.cli import cli; cli()",
+#	"--output", "${TARGET.abspath}",
+#	"--quiet", "--html", "--include-log", "--no-frames",
+#	"--name", "NVDA", "--url", "https://www.nvaccess.org/",
+#	"*.py", "appModules", "brailleDisplayDrivers", r"comInterfaces\__init__.py",
+#	"config", "contentRecog", "extensionPoints", "globalPlugins", "gui", "mathPres", "NVDAObjects",
+#	"speechDictHandler", "synthDrivers", "textInfos", "virtualBuffers",
+#]])
+
+#env.Alias('devDocs', [devGuide, devDocs_nvda])
+#env.Clean('devDocs', [devGuide, devDocs_nvda])
 
 pot = env.Command(outputDir.File("%s.pot" % outFilePrefix),
 	# Don't use sourceDir as the source, as this depends on comInterfaces and nvdaHelper.

site_scons/site_tools/doxygen.py

@@ -23,10 +23,8 @@
 import os.path
 import glob
 from fnmatch import fnmatch
-try:
-	import _winreg as winreg # Python 2.7 import
-except:
-	import winreg # python 3 import
+from functools import reduce
+import winreg
 
 def fetchDoxygenPath():
 	try:
@@ -74,7 +72,7 @@ def append_data(data, key, new_data, token):
          key_token = False
       else:
          if token == "+=":
-            if not data.has_key(key):
+            if key not in data:
                data[key] = list()
          elif token == "=":
             data[key] = list()
@@ -90,7 +88,8 @@ def append_data(data, key, new_data, token):
          append_data( data, key, new_data, '\\' )
 
    # compress lists of len 1 into single strings
-   for (k, v) in data.items():
+   # Wrap items into a list, since we're mutating the dictionary
+   for (k, v) in list(data.items()):
       if len(v) == 0:
          data.pop(k)
 
@@ -121,7 +120,8 @@ def DoxySourceScan(node, env, path):
 
    sources = []
 
-   data = DoxyfileParse(node.get_contents())
+   with open(node.abspath) as contents:
+      data = DoxyfileParse(contents)
 
    if data.get("RECURSIVE", "NO") == "YES":
       recursive = True
@@ -149,7 +149,7 @@ def DoxySourceScan(node, env, path):
             for pattern in file_patterns:
                sources.extend(glob.glob("/".join([node, pattern])))
 
-   sources = map( lambda path: env.File(path), sources )
+   sources = [env.File(path) for path in sources]
    return sources
 
 
@@ -168,13 +168,14 @@ def DoxyEmitter(source, target, env):
       "XML": ("NO", "xml"),
    }
 
-   data = DoxyfileParse(source[0].get_contents())
+   with open(source[0].abspath) as contents:
+      data = DoxyfileParse(contents)
 
    targets = []
    out_dir = source[0].Dir(data.get("OUTPUT_DIRECTORY", "."))
 
    # add our output locations
-   for (k, v) in output_formats.items():
+   for (k, v) in list(output_formats.items()):
       if data.get("GENERATE_" + k, v[0]) == "YES":
          targets.append(out_dir.Dir(v[1]))
 

source/comInterfaces_sconscript

@@ -16,6 +16,8 @@ Import(
 	'env',
 )
 
+import importlib.util
+
 def interfaceAction(target,source,env):
 	clsid=env.get('clsid')
 	if clsid:
@@ -46,12 +48,12 @@ COM_INTERFACES = {
 	"FlashAccessibility.py": "typelibs/FlashAccessibility.tlb",
 }
 
-for k,v in COM_INTERFACES.iteritems():
+for k,v in COM_INTERFACES.items():
 	targets=[Dir('comInterfaces').File(k),
 		# This buillds a .pyc file as well.
-		Dir('comInterfaces').File(k + "c")]
+		Dir('comInterfaces').File(importlib.util.cache_from_source(k))]
 	source=clsid=majorVersion=None
-	if isinstance(v,basestring):
+	if isinstance(v, str):
 		env.comtypesInterface(targets,v)
 	else:
 		env.comtypesInterface(targets,Dir('comInterfaces').File('__init__.py'),clsid=v[0],majorVersion=v[1],minorVersion=v[2])