Skip to content
Permalink
Browse files

Update Scons build environment to run on Python 3 (#9667)

* Update batch script and python wrapper for scons

* * Update copyright
* Remove obsolete _winreg import
* No longer pass unicode=True to gettext.install

* * sconstruct: xrange>range
* comInterfaces_sconscript: iteritems>items
* comInterfaces_sconscript: basestring>str
* nvdaHelper/archBuild_sconscript: xrange>range
* Liblouis sconscript: file>open
* cldrDict_sconscript: iteritems>items

* Disable developer documentation for now

* comInterfaces_sconscript: make sure that the byte compiled targets are picked up correctly by scons

* Fix espeak initialization and paths

* sconstruct: file>open

* Convert doxygen script to Python 3 and fix doxygen site tool

* Fix pot files creation

* Fix several issues to the dist builder

* Issue errors about str(bytes_instance), str(bytearray_instance)

* Update readme
  • Loading branch information...
leonardder authored and michaelDCurran committed Jun 11, 2019
1 parent f1f823e commit 840dce5bbfe4ab7e3bfb22e30bed31e66ce243a2
@@ -29,7 +29,7 @@ def createCLDRAnnotationsDict(sources, dest):
assert cldrDict, "cldrDict is empty"
with codecs.open(dest, "w", "utf_8_sig", errors="replace") as dictFile:
dictFile.write(u"symbols:\r\n")
for pattern, description in cldrDict.iteritems():
for pattern, description in cldrDict.items():
dictFile.write(u"{pattern}\t{description}\tsome\r\n".format(
pattern=pattern,
description=description
@@ -119,7 +119,7 @@ NVDAToCLDRLocales = {

annotationsDir = env.Dir("include/cldr-emoji-annotation/annotations")
annotationsDerivedDir = env.Dir("include/cldr-emoji-annotation/annotationsDerived")
for destLocale, sourceLocales in NVDAToCLDRLocales.iteritems():
for destLocale, sourceLocales in NVDAToCLDRLocales.items():
cldrSources = []
# First add all annotations, then the derived ones.
for sourceLocale in sourceLocales:
@@ -34,7 +34,7 @@ def clsidStringToCLSIDDefine(clsidString):
"0x"+d[0:8],
"0x"+d[8:12],
"0x"+d[12:16],
"{%s}"%(",".join("0x"+d[x:x+2] for x in xrange(16,32,2)))
"{%s}"%(",".join("0x"+d[x:x+2] for x in range(16,32,2)))
)

def COMProxyDllBuilder(env,target,source,proxyClsid):
@@ -59,7 +59,7 @@ def espeak_compilePhonemeData_buildAction(target,source,env):
# Unfortunately, there's no way we can flush it or use a different stream
# because our eSpeak statically links the CRT.
espeak=AutoFreeCDLL(espeakLib[0].abspath)
espeak.espeak_ng_InitializePath(espeakRepo.abspath)
espeak.espeak_ng_InitializePath(os.fsencode(espeakRepo.abspath))
espeak.espeak_ng_CompileIntonation(None,None)
espeak.espeak_ng_CompilePhonemeData(22050,None,None)
espeak.espeak_Terminate()
@@ -72,14 +72,14 @@ def espeak_compileDict_buildAction(target,source,env):
# Unfortunately, there's no way we can flush it or use a different stream
# because our eSpeak statically links the CRT.
espeak=AutoFreeCDLL(espeakLib[0].abspath)
espeak.espeak_Initialize(0,0,target[0].Dir('..').abspath,0x8000)
espeak.espeak_Initialize(0,0,os.fsencode(target[0].Dir('..').abspath),0x8000)
try:
lang=source[0].name.split('_')[0]
v=espeak_VOICE(languages=lang+'\x00')
lang=source[0].name.split('_')[0].encode()
v=espeak_VOICE(languages=lang+b'\x00')
if espeak.espeak_SetVoiceByProperties(ctypes.byref(v))!=0:
print("espeak_compileDict_action: failed to switch to language %s"%lang)
return 1
dictPath=os.path.split(source[0].abspath)[0]+'/'
dictPath=os.fsencode(os.path.split(source[0].abspath)[0]+'/')
if espeak.espeak_ng_CompileDictionary(dictPath,None,0,None)!=0:
print("espeak_compileDict_action: failed to compile dictionary for language %s"%lang)
return
@@ -28,7 +28,7 @@ signExec=env['signExec'] if env['certFile'] else None
RE_AC_INIT = re.compile(r"^AC_INIT\(\[(?P<package>.*)\], \[(?P<version>.*)\], \[(?P<bugReport>.*)\], \[(?P<tarName>.*)\], \[(?P<url>.*)\]\)")
def getLouisVersion():
# Get the version from configure.ac.
with file(louisRootDir.File("configure.ac").abspath) as f:
with open(louisRootDir.File("configure.ac").abspath) as f:
for line in f:
m = RE_AC_INIT.match(line)
if m:
@@ -91,7 +91,7 @@ For reference, the following dependencies are included in Git submodules:
### Other Dependencies
These dependencies are not included in Git submodules, but aren't needed by most people.

* To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.stack.nl/~dimitri/doxygen/download.html), version 1.7.3:
* To generate developer documentation for nvdaHelper: [Doxygen Windows installer](http://www.doxygen.nl/download.html), version 1.8.15:

## Preparing the Source Tree
Before you can run the NVDA source code, you must prepare the source tree.
@@ -161,13 +161,14 @@ scons launcher

The archive will be placed in the output directory.

To generate developer documentation, type:
To generate the NVDA developer guide, type:

```
scons devDocs
scons developerGuide
```

The developer docs will be placed in the `devDocs` folder in the output directory.
The developer guide will be placed in the `devDocs` folder in the output directory.
Note that the Python 3 sources of NVDA currently do not support building NVDA developer documentation using the `scons devDocs` command.

To generate developer documentation for nvdaHelper (not included in the devDocs target):

@@ -5,8 +5,8 @@ rem Instead, find the python launcher (installed by python 3)
where py 1>nul 2>&1
if "%ERRORLEVEL%" == "0" (
rem Python launcher is present in the PATH
rem Call python 2.7 for 32 bits
py -2.7-32 "%~dp0\scons.py" %*
rem Call python 3.7 for 32 bits
py -3.7-32 "%~dp0\scons.py" %*
) else (
rem Python registers itself with the .py extension, so call scons.py.
"%~dp0\scons.py" %*
@@ -5,7 +5,7 @@
import os
import platform
# Variables for storing required version of Python, and the version which is used to run this script.
requiredPythonMajor ="2"
requiredPythonMajor ="3"
requiredPythonMinor = "7"
requiredPythonArchitecture = "32bit"
installedPythonMajor = str(sys.version_info.major)
@@ -1,7 +1,7 @@
###
#This file is a part of the NVDA project.
#URL: https://www.nvaccess.org/
#Copyright 2010-2017 NV Access Limited.
#Copyright 2010-2019 NV Access Limited, Babbage B.V.
#This program is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License version 2.0, as published by
#the Free Software Foundation.
@@ -15,9 +15,9 @@
import sys
import os
import time
import _winreg
from glob import glob
import sourceEnv
import importlib.util

def recursiveCopy(env,targetDir,sourceDir):
targets=[]
@@ -35,7 +35,7 @@ def recursiveCopy(env,targetDir,sourceDir):

# Import NVDA's versionInfo module.
import gettext
gettext.install("nvda", unicode=True)
gettext.install("nvda")
sys.path.append("source")
import versionInfo
del sys.path[-1]
@@ -72,14 +72,21 @@ vars.Add("certPassword", "The password for the private key in the signing certif
vars.Add("certTimestampServer", "The URL of the timestamping server to use to timestamp authenticode signatures", "")
vars.Add(PathVariable("outputDir", "The directory where the final built archives and such will be placed", "output",PathVariable.PathIsDirCreate))
vars.Add(ListVariable("nvdaHelperDebugFlags", "a list of debugging features you require", 'none', ["debugCRT","RTC","analyze"]))
vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in xrange(60)]))
vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in range(60)]))
if "tests" in COMMAND_LINE_TARGETS:
vars.Add("unitTests", "A list of unit tests to run", "")
if "systemTests" in COMMAND_LINE_TARGETS:
vars.Add("filter", "A filter for the name of the system test(s) to run. Wildcards accepted.", "")

#Base environment for this and sub sconscripts
env = Environment(variables=vars,HOST_ARCH='x86',tools=["textfile","gettextTool","t2t",keyCommandsDocTool,'doxygen','recursiveInstall'])
env = Environment(variables=vars,HOST_ARCH='x86',tools=[
"textfile",
"gettextTool",
"t2t"
,keyCommandsDocTool,
"doxygen",
"recursiveInstall"
])

# speed up subsiquent runs by checking timestamps of targets and dependencies, and only using md5 if timestamps differ.
env.Decider('MD5-timestamp')
@@ -139,7 +146,7 @@ def signExec(target,source,env):
#sys.exit(1)
# #3795: signtool can quite commonly fail with timestamping, so allow it to try up to 3 times with a 1 second delay between each try.
res=0
for count in xrange(3):
for count in range(3):
res=env.Execute([signExecCmd+[target[0].abspath]])
if not res:
return 0 # success
@@ -228,31 +235,41 @@ def NVDADistGenerator(target, source, env, for_signature):
# We don't do this using normal scons mechanisms because we want it to be cleaned up immediately after this builder
# and py2exe will cause bytecode files to be created for it which scons doesn't know about.
updateVersionType = env["updateVersionType"] or None
action = [lambda target, source, env: file(buildVersionFn, "w").write(
'version = {version!r}\r\n'
'publisher = {publisher!r}\r\n'
'updateVersionType = {updateVersionType!r}\r\n'
'version_build = {version_build!r}\r\n'
.format(version=version, publisher=publisher, updateVersionType=updateVersionType,version_build=version_build))]
# Any '\n' characters written are translated to the system default line separator, os.linesep.
action = [lambda target, source, env: open(buildVersionFn, "w", encoding="utf-8").write(
'version = {version!r}\n'
'publisher = {publisher!r}\n'
'updateVersionType = {updateVersionType!r}\n'
'version_build = {version_build!r}\n'
.format(version=version, publisher=publisher, updateVersionType=updateVersionType,version_build=version_build)
)
# In Python 3 write returns the number of characters written,
# which scons treats as an error code.
and None]

buildCmd = ["cd", source[0].path, "&&",
sys.executable]
if release:
buildCmd.append("-O")
# Issue errors about str(bytes_instance), str(bytearray_instance)
buildCmd.append("-bb")
buildCmd.extend(("setup.py", "build", "--build-base", buildDir.abspath,
"py2exe", "--dist-dir", target[0].abspath))
if release:
buildCmd.append("-O1")
action.append(buildCmd)

if env.get("uiAccess"):
buildCmd.append("--enable-uiAccess")

action.append(buildCmd)

if certFile:
for prog in "nvda_noUIAccess.exe", "nvda_uiAccess.exe", "nvda_slave.exe", "nvda_eoaProxy.exe":
action.append(lambda target,source,env, progByVal=prog: signExec([target[0].File(progByVal)],source,env))

for ext in "", "c", "o":
action.append(Delete(buildVersionFn + ext))
action.extend((
Delete(buildVersionFn),
Delete(importlib.util.cache_from_source(buildVersionFn))
))

return action
env["BUILDERS"]["NVDADist"] = Builder(generator=NVDADistGenerator, target_factory=Dir)
@@ -353,7 +370,7 @@ def makePot(target, source, env):
# Tweak the headers.
potFn = str(target[0])
tmpFn = "%s.tmp" % potFn
with file(potFn, "rt") as inp, file(tmpFn, "wt") as out:
with open(potFn, "rt") as inp, open(tmpFn, "wt") as out:
for lineNum, line in enumerate(inp):
if lineNum == 1:
line = "# %s\n" % versionInfo.copyright
@@ -371,19 +388,19 @@ devDocs_nvdaHelper=env.Command(devDocsOutputDir.Dir('nvdaHelper'),devDocs_nvdaHe
env.Alias('devDocs_nvdaHelper', devDocs_nvdaHelper)
env.Clean('devDocs_nvdaHelper', devDocs_nvdaHelper)

devDocs_nvda = env.Command(devDocsOutputDir.Dir("nvda"), None, [[
"cd", sourceDir.path, "&&",
sys.executable, "-c", "import sourceEnv; from epydoc.cli import cli; cli()",
"--output", "${TARGET.abspath}",
"--quiet", "--html", "--include-log", "--no-frames",
"--name", "NVDA", "--url", "https://www.nvaccess.org/",
"*.py", "appModules", "brailleDisplayDrivers", r"comInterfaces\__init__.py",
"config", "contentRecog", "extensionPoints", "globalPlugins", "gui", "mathPres", "NVDAObjects",
"speechDictHandler", "synthDrivers", "textInfos", "virtualBuffers",
]])

env.Alias('devDocs', [devGuide, devDocs_nvda])
env.Clean('devDocs', [devGuide, devDocs_nvda])
#devDocs_nvda = env.Command(devDocsOutputDir.Dir("nvda"), None, [[
# "cd", sourceDir.path, "&&",
# sys.executable, "-c", "import sourceEnv; from epydoc.cli import cli; cli()",
# "--output", "${TARGET.abspath}",
# "--quiet", "--html", "--include-log", "--no-frames",
# "--name", "NVDA", "--url", "https://www.nvaccess.org/",
# "*.py", "appModules", "brailleDisplayDrivers", r"comInterfaces\__init__.py",
# "config", "contentRecog", "extensionPoints", "globalPlugins", "gui", "mathPres", "NVDAObjects",
# "speechDictHandler", "synthDrivers", "textInfos", "virtualBuffers",
#]])

#env.Alias('devDocs', [devGuide, devDocs_nvda])
#env.Clean('devDocs', [devGuide, devDocs_nvda])

pot = env.Command(outputDir.File("%s.pot" % outFilePrefix),
# Don't use sourceDir as the source, as this depends on comInterfaces and nvdaHelper.
@@ -23,10 +23,8 @@
import os.path
import glob
from fnmatch import fnmatch
try:
import _winreg as winreg # Python 2.7 import
except:
import winreg # python 3 import
from functools import reduce
import winreg

def fetchDoxygenPath():
try:
@@ -74,7 +72,7 @@ def append_data(data, key, new_data, token):
key_token = False
else:
if token == "+=":
if not data.has_key(key):
if key not in data:
data[key] = list()
elif token == "=":
data[key] = list()
@@ -90,7 +88,8 @@ def append_data(data, key, new_data, token):
append_data( data, key, new_data, '\\' )

# compress lists of len 1 into single strings
for (k, v) in data.items():
# Wrap items into a list, since we're mutating the dictionary
for (k, v) in list(data.items()):
if len(v) == 0:
data.pop(k)

@@ -121,7 +120,8 @@ def DoxySourceScan(node, env, path):

sources = []

data = DoxyfileParse(node.get_contents())
with open(node.abspath) as contents:
data = DoxyfileParse(contents)

if data.get("RECURSIVE", "NO") == "YES":
recursive = True
@@ -149,7 +149,7 @@ def DoxySourceScan(node, env, path):
for pattern in file_patterns:
sources.extend(glob.glob("/".join([node, pattern])))

sources = map( lambda path: env.File(path), sources )
sources = [env.File(path) for path in sources]
return sources


@@ -168,13 +168,14 @@ def DoxyEmitter(source, target, env):
"XML": ("NO", "xml"),
}

data = DoxyfileParse(source[0].get_contents())
with open(source[0].abspath) as contents:
data = DoxyfileParse(contents)

targets = []
out_dir = source[0].Dir(data.get("OUTPUT_DIRECTORY", "."))

# add our output locations
for (k, v) in output_formats.items():
for (k, v) in list(output_formats.items()):
if data.get("GENERATE_" + k, v[0]) == "YES":
targets.append(out_dir.Dir(v[1]))

@@ -16,6 +16,8 @@ Import(
'env',
)

import importlib.util

def interfaceAction(target,source,env):
clsid=env.get('clsid')
if clsid:
@@ -46,12 +48,12 @@ COM_INTERFACES = {
"FlashAccessibility.py": "typelibs/FlashAccessibility.tlb",
}

for k,v in COM_INTERFACES.iteritems():
for k,v in COM_INTERFACES.items():
targets=[Dir('comInterfaces').File(k),
# This buillds a .pyc file as well.
Dir('comInterfaces').File(k + "c")]
Dir('comInterfaces').File(importlib.util.cache_from_source(k))]
source=clsid=majorVersion=None
if isinstance(v,basestring):
if isinstance(v, str):
env.comtypesInterface(targets,v)
else:
env.comtypesInterface(targets,Dir('comInterfaces').File('__init__.py'),clsid=v[0],majorVersion=v[1],minorVersion=v[2])

0 comments on commit 840dce5

Please sign in to comment.
You can’t perform that action at this time.