diff --git a/.gitignore b/.gitignore index 1de6c3df665..bec991f09bc 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ extras/controllerClient/x64 source/_buildVersion.py user_docs/*/keyCommands.t2t output +testOutput source/brailleDisplayDrivers/handyTech ./developerGuide.html user_docs/build.t2tConf diff --git a/appveyor.yml b/appveyor.yml index f2b64ef5fcf..8445c52b001 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,6 +20,7 @@ environment: init: - ps: | + iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) if ($env:APPVEYOR_REPO_TAG_NAME -and $env:APPVEYOR_REPO_TAG_NAME.StartsWith("release-")) { # Strip "release-" prefix. $version = $env:APPVEYOR_REPO_TAG_NAME.Substring(8) @@ -91,8 +92,8 @@ build_script: Set-AppveyorBuildVariable "sconsArgs" $sconsArgs - 'echo scons args: %sconsArgs%' - py scons.py source %sconsArgs% - # We don't need launcher to run tests, so run the tests before launcher. - - py scons.py tests %sconsArgs% + # We don't need launcher to run checkPot, so run the checkPot before launcher. + - py scons.py checkPot %sconsArgs% # The pot gets built by tests, but we don't actually need it as a build artifact. - del output\*.pot - 'echo scons output targets: %sconsOutTargets%' @@ -113,6 +114,55 @@ build_script: - 7z a -tzip -r ..\output\symbols.zip *.dl_ *.ex_ *.pd_ - cd .. +before_test: + - py -m pip install robotframework robotremoteserver pyautogui nose + - mkdir testOutput + - mkdir testOutput\unit + - mkdir testOutput\system + - ps: | + $errorCode=0 + $nvdaLauncherFile=".\output\nvda" + if(!$env:release) { + $nvdaLauncherFile+="_snapshot" + } + $nvdaLauncherFile+="_${env:version}.exe" + echo NVDALauncherFile: $NVDALauncherFile + $outputDir=$(resolve-path .\testOutput) + $installerLogFilePath="$outputDir\nvda_install.log" + $installerProcess=start-process -FilePath "$nvdaLauncherFile" -ArgumentList "--install-silent --debug-logging --log-file $installerLogFilePath" -passthru + try { + $installerProcess | wait-process -Timeout 180 + $errorCode=$installerProcess.ExitCode + } catch { + echo NVDA installer process timed out + $errorCode=1 + } + Push-AppveyorArtifact $installerLogFilePath + if($errorCode -ne 0) { $host.SetShouldExit($errorCode) } + +test_script: + - ps: | + $errorCode=0 + $outDir = (Resolve-Path .\testOutput\unit\) + $unitTestsXml = "$outDir\unitTests.xml" + py -m nose --with-xunit --xunit-file="$unitTestsXml" ./tests/unit + if($LastExitCode -ne 0) { $errorCode=$LastExitCode } + Push-AppveyorArtifact $unitTestsXml + $wc = New-Object 'System.Net.WebClient' + $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", $unitTestsXml) + if($errorCode -ne 0) { $host.SetShouldExit($errorCode) } + - ps: | + $testOutput = (Resolve-Path .\testOutput\) + $systemTestOutput = (Resolve-Path "$testOutput\system") + $testSource = "./tests/system" + py -m robot --loglevel DEBUG -d $systemTestOutput -x systemTests.xml -v whichNVDA:installed -P "$testSource/libraries" "$testSource" + Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip" + Push-AppveyorArtifact "$testOutput\systemTestResult.zip" + if($LastExitCode -ne 0) { $errorCode=$LastExitCode } + $wc = New-Object 'System.Net.WebClient' + $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$systemTestOutput\systemTests.xml")) + if($errorCode -ne 0) { $host.SetShouldExit($errorCode) } + artifacts: - path: output\* - path: output\*\* diff --git a/readme.md b/readme.md index 51629f54bcc..b6efaadbffd 100644 --- a/readme.md +++ b/readme.md @@ -230,3 +230,16 @@ To run only the translatable string checks (which check that all translatable st ``` scons checkPot ``` + +You may also use scons to run the system tests, though this will still rely on having set up the dependencies (see `tests/system/readme.md`). + +``` +scons systemTests +``` + +To run only specific system tests, specify them using the `filter` variable on the command line. +This filter accepts wildcard characters. + +``` +scons systemTests filter="Read welcome dialog" +``` \ No newline at end of file diff --git a/sconstruct b/sconstruct index 53fa8c0603f..7151932e352 100755 --- a/sconstruct +++ b/sconstruct @@ -75,6 +75,8 @@ vars.Add(ListVariable("nvdaHelperDebugFlags", "a list of debugging features you vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in xrange(60)])) if "tests" in COMMAND_LINE_TARGETS: vars.Add("unitTests", "A list of unit tests to run", "") +if "systemTests" in COMMAND_LINE_TARGETS: + vars.Add("filter", "A filter for the name of the system test(s) to run. Wildcards accepted.", "") #Base environment for this and sub sconscripts env = Environment(variables=vars,HOST_ARCH='x86',tools=["textfile","gettextTool","t2t",keyCommandsDocTool,'doxygen','recursiveInstall']) diff --git a/source/core.py b/source/core.py index 791a5a585ce..b5f5b9a951b 100644 --- a/source/core.py +++ b/source/core.py @@ -29,6 +29,11 @@ from logHandler import log import addonHandler +import extensionPoints + +# inform those who want to know that NVDA has finished starting up. +postNvdaStartup = extensionPoints.Action() + PUMP_MAX_DELAY = 10 #: The thread identifier of the main thread. @@ -470,6 +475,7 @@ def Notify(self): log.debug("initializing updateCheck") updateCheck.initialize() log.info("NVDA initialized") + postNvdaStartup.notify() log.debug("entering wx application main loop") app.MainLoop() diff --git a/source/nvda.pyw b/source/nvda.pyw index d0d78e87e16..d023322cb4c 100755 --- a/source/nvda.pyw +++ b/source/nvda.pyw @@ -194,6 +194,8 @@ if globalVars.appArgs.debugLogging: logLevel=log.DEBUG logHandler.initialize() logHandler.log.setLevel(logLevel) +if logLevel is log.DEBUG: + log.debug("Provided arguments: {}".format(sys.argv[1:])) log.info("Starting NVDA") log.debug("Debug level logging enabled") diff --git a/source/speech.py b/source/speech.py index 956e353b1c5..14d1988887b 100755 --- a/source/speech.py +++ b/source/speech.py @@ -496,7 +496,7 @@ def speak(speechSequence,symbolLevel=None): import speechViewer if speechViewer.isActive: for item in speechSequence: - if isinstance(item,basestring): + if isinstance(item, basestring): speechViewer.appendText(item) global beenCanceled, curWordChars curWordChars=[] diff --git a/tests/sconscript b/tests/sconscript index c9450fde5ce..68ff8bad232 100644 --- a/tests/sconscript +++ b/tests/sconscript @@ -20,6 +20,11 @@ unitTests = env.SConscript("unit/sconscript", exports=["env"]) env.Depends(unitTests, sourceDir) env.AlwaysBuild(unitTests) +systemTests = env.SConscript("system/sconscript", exports=["env"]) +env.Depends(systemTests, sourceDir) +env.AlwaysBuild(systemTests) +env.Alias("systemTests", systemTests) + def checkPotAction(target, source, env): return checkPot.checkPot(source[0].abspath) checkPotTarget = env.Command("checkPot", pot, checkPotAction) @@ -31,10 +36,13 @@ env.Alias("checkPot", checkPotTarget) # If specific tests are explicitly specified, only run those. explicitUnitTests = env.get("unitTests") explicitCheckPot = "checkPot" in COMMAND_LINE_TARGETS -explicit = explicitUnitTests or explicitCheckPot +explicitSystemTests = "systemTests" in COMMAND_LINE_TARGETS +explicit = explicitUnitTests or explicitCheckPot or explicitSystemTests tests = [] if not explicit or explicitUnitTests: tests.append(unitTests) if not explicit or explicitCheckPot: tests.append(checkPotTarget) +if explicit and explicitSystemTests: # only run system tests explicitly + tests.append(systemTests) env.Alias("tests", tests) diff --git a/tests/system/.gitignore b/tests/system/.gitignore new file mode 100644 index 00000000000..cec98301fd4 --- /dev/null +++ b/tests/system/.gitignore @@ -0,0 +1 @@ +nvdaProfile/ diff --git a/tests/system/NVDA Core/startupShutdownNVDA.robot b/tests/system/NVDA Core/startupShutdownNVDA.robot new file mode 100644 index 00000000000..edd6050aba5 --- /dev/null +++ b/tests/system/NVDA Core/startupShutdownNVDA.robot @@ -0,0 +1,57 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html +*** Settings *** +Documentation Basic start and exit tests +Default Tags NVDA smoke test + +Library OperatingSystem +Library Process +Library sendKey.py +Library nvdaRobotLib.py +Library helperLib.py + +Test Setup start NVDA standard-dontShowWelcomeDialog.ini +Test Teardown quit NVDA + +Variables variables.py + +*** Test Cases *** +Starts + [Documentation] Ensure that NVDA can start + process should be running nvdaAlias + +Quits from keyboard + [Documentation] Starts NVDA and ensures that it can be quit using the keyboard + [Setup] start NVDA standard-doShowWelcomeDialog.ini + + ${Welcome dialog title} = catenate double space Welcome to NVDA dialog + wait for specific speech ${Welcome dialog title} + wait for speech to finish + sleep 1 # the dialog is not always receiving the enter keypress, wait a little longer for it + send key enter + + ${Exit NVDA dialog} = catenate double space Exit NVDA dialog + send key insert q + ${INDEX} = wait for specific speech ${Exit NVDA dialog} + + wait for speech to finish + ${actual speech} = get speech from index until now ${INDEX} + assert strings are equal ${actual speech} ${QUIT_DIALOG_TEXT} + sleep 1 # the dialog is not always receiving the enter keypress, wait a little longer for it + send key enter + wait for process nvdaAlias timeout=10 sec + process should be stopped nvdaAlias + +Read welcome dialog + [Documentation] Ensure that the welcome dialog can be read in full + [Setup] start NVDA standard-doShowWelcomeDialog.ini + + ${Welcome dialog title} = catenate double space Welcome to NVDA dialog + ${INDEX} = wait for specific speech ${Welcome dialog title} + wait for speech to finish + ${actual speech} = get speech from index until now ${INDEX} + assert strings are equal ${actual speech} ${WELCOME_DIALOG_TEXT} + sleep 1 # the dialog is not always receiving the enter keypress, wait a little longer for it + send key enter diff --git a/tests/system/NVDA Core/variables.py b/tests/system/NVDA Core/variables.py new file mode 100644 index 00000000000..edf4fc4dc32 --- /dev/null +++ b/tests/system/NVDA Core/variables.py @@ -0,0 +1,13 @@ +WELCOME_DIALOG_TEXT = ( + "Welcome to NVDA dialog Welcome to NVDA! Most commands for controlling NVDA require you to hold " + "down the NVDA key while pressing other keys. By default, the numpad Insert and main Insert keys " + "may both be used as the NVDA key. You can also configure NVDA to use the Caps Lock as the NVDA " + "key. Press NVDA plus n at any time to activate the NVDA menu. From this menu, you can configure " + "NVDA, get help and access other NVDA functions.\n" + "Options grouping\n" + "Keyboard layout: combo box desktop collapsed Alt plus k" +) +QUIT_DIALOG_TEXT = ( + "Exit NVDA dialog\n" + "What would you like to do? combo box Exit collapsed Alt plus d" +) \ No newline at end of file diff --git a/tests/system/libraries/helperLib.py b/tests/system/libraries/helperLib.py new file mode 100644 index 00000000000..bd791983b03 --- /dev/null +++ b/tests/system/libraries/helperLib.py @@ -0,0 +1,34 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""This file provides general robot library functions for system tests. +This is in contrast with nvdaRobotLib.py which contains helpers related to starting and stopping NVDA for system +tests, or with systemTestSpy which contains methods for extracting information about NVDA's behaviour during system +tests. +""" +from robot.libraries.BuiltIn import BuiltIn +builtIn = BuiltIn() # type: BuiltIn + +def assert_strings_are_equal( actual, expected, ignore_case=False): + try: + builtIn.should_be_equal_as_strings( + actual, + expected, + msg="Actual speech != Expected speech", + ignore_case=ignore_case + ) + except AssertionError: + builtIn.log( + "repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format( + ignore_case, + repr(actual), + repr(expected) + ) + ) + raise + + +def catenate_double_space(*args): + return " ".join(args) \ No newline at end of file diff --git a/tests/system/libraries/nvdaRobotLib.py b/tests/system/libraries/nvdaRobotLib.py new file mode 100644 index 00000000000..c58964deb19 --- /dev/null +++ b/tests/system/libraries/nvdaRobotLib.py @@ -0,0 +1,227 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""This file provides robot library functions for NVDA system tests. It contains helper methods for system tests, +most specifically related to the setup for, starting of, quiting of, and cleanup of, NVDA. This is in contrast with +the systemTestSpy.py file, which provides library functions related to monitoring / asserting NVDA output. +""" +# imported methods start with underscore (_) so they don't get imported into robot files as keywords +from os.path import join as _pJoin, abspath as _abspath, expandvars as _expandvars +import tempfile +import sys +from robotremoteserver import test_remote_server as _testRemoteServer, stop_remote_server as _stopRemoteServer +from robot.libraries.BuiltIn import BuiltIn +from robot.libraries.OperatingSystem import OperatingSystem +from robot.libraries.Process import Process +from systemTestUtils import _blockUntilConditionMet + +builtIn = BuiltIn() # type: BuiltIn +process = builtIn.get_library_instance('Process') # type: Process +opSys = builtIn.get_library_instance('OperatingSystem') # type: OperatingSystem + +spyServerPort = 8270 # is `registered by IANA` for remote server usage. Two ASCII values:'RF' +spyServerURI = 'http://127.0.0.1:{}'.format(spyServerPort) +spyAlias = "nvdaSpy" + +# robot is expected to be run from the NVDA repo root directory. We want all repo specific +# paths to be relative to this. This would allow us to change where it is run from if we decided to. +repoRoot = _abspath("./") +whichNVDA = builtIn.get_variable_value("${whichNVDA}", "source") +if whichNVDA == "source": + NVDACommandPathToCheckExists = _pJoin(repoRoot, "source/nvda.pyw") + baseNVDACommandline = "pythonw "+NVDACommandPathToCheckExists +elif whichNVDA == "installed": + NVDACommandPathToCheckExists = _pJoin(_expandvars('%PROGRAMFILES%'),'nvda','nvda.exe') + baseNVDACommandline='"%s"' % NVDACommandPathToCheckExists +else: + raise AssertionError("robot should be given argument `-v whichNVDA [source|installed]") + +# Paths +systemTestSourceDir = _pJoin(repoRoot, "tests", "system") +tempDir = tempfile.gettempdir() +opSys.directory_should_exist(tempDir) +nvdaProfileWorkingDir = _pJoin(tempDir, "nvdaProfile") +nvdaLogFilePath = _pJoin(nvdaProfileWorkingDir, 'nvda.log') +systemTestSpyAddonName = "systemTestSpy" +testSpyPackageDest = _pJoin(nvdaProfileWorkingDir, "globalPlugins") +outDir = builtIn.get_variable_value("${OUTPUT DIR}") +testOutputNvdaLogsDir = _pJoin(outDir, "nvdaTestRunLogs") + + +def _findDepPath(depFileName, searchPaths): + import os + for path in searchPaths: + filePath = _pJoin(path, depFileName+".py") + if os.path.isfile(filePath): + return filePath + elif os.path.isfile(_pJoin(path, depFileName, "__init__.py")): + return _pJoin(path, depFileName) + raise AssertionError("Unable to find required system test spy dependency: {}".format(depFileName)) + +# relative to the python path +requiredPythonImportsForSystemTestSpyPackage = [ + r"robotremoteserver", + r"SimpleXMLRPCServer", + r"xmlrpclib", +] + +def _createNvdaSpyPackage(): + import os + searchPaths = sys.path + profileSysTestSpyPackageStagingDir = _pJoin(tempDir, systemTestSpyAddonName) + # copy in required dependencies, the addon will modify the python path + # to point to this sub dir + spyPackageLibsDir = _pJoin(profileSysTestSpyPackageStagingDir, "libs") + opSys.create_directory(spyPackageLibsDir) + for lib in requiredPythonImportsForSystemTestSpyPackage: + libSource = _findDepPath(lib, searchPaths) + if os.path.isdir(libSource): + opSys.copy_directory(libSource, spyPackageLibsDir) + elif os.path.isfile(libSource): + opSys.copy_file(libSource, spyPackageLibsDir) + + opSys.copy_file( + _pJoin(systemTestSourceDir, "libraries", systemTestSpyAddonName+".py"), + _pJoin(profileSysTestSpyPackageStagingDir, "__init__.py") + ) + opSys.copy_file( + _pJoin(systemTestSourceDir, "libraries", "systemTestUtils.py"), + profileSysTestSpyPackageStagingDir + ) + return profileSysTestSpyPackageStagingDir + +def _createTestIdFileName(name): + suiteName = builtIn.get_variable_value("${SUITE NAME}") + testName = builtIn.get_variable_value("${TEST NAME}") + outputFileName = "{suite}-{test}-{name}".format( + suite=suiteName, + test=testName, + name=name, + ).replace(" ", "_") + return outputFileName + +class nvdaRobotLib(object): + + def __init__(self): + self.nvdaSpy = None + self.nvdaHandle = None + + def setup_nvda_profile(self, settingsFileName): + builtIn.log("Copying files into NVDA profile") + opSys.copy_file( + _pJoin(systemTestSourceDir, "nvdaSettingsFiles", settingsFileName), + _pJoin(nvdaProfileWorkingDir, "nvda.ini") + ) + # create a package to use as the globalPlugin + opSys.move_directory( + _createNvdaSpyPackage(), + _pJoin(testSpyPackageDest, systemTestSpyAddonName) + ) + # install the test spy speech synth + opSys.copy_file( + _pJoin(systemTestSourceDir, "libraries", "speechSpy.py"), + _pJoin(nvdaProfileWorkingDir, "synthDrivers", "speechSpy.py") + ) + + def teardown_nvda_profile(self): + builtIn.log("Removing files from NVDA profile") + opSys.remove_file( + _pJoin(nvdaProfileWorkingDir, "nvda.ini") + ) + opSys.remove_directory( + testSpyPackageDest, + recursive=True + ) + opSys.remove_file( + _pJoin(nvdaProfileWorkingDir, "synthDrivers", "speechSpy.py") + ) + + def _startNVDAProcess(self): + """Start NVDA. + Use debug logging, replacing any current instance, using the system test profile directory + """ + opSys.create_directory(testOutputNvdaLogsDir) + opSys.file_should_exist(NVDACommandPathToCheckExists, "Unable to start NVDA unless path exists.") + self.nvdaHandle = handle = process.start_process( + "{baseNVDACommandline} --debug-logging -r -c \"{nvdaProfileDir}\" --log-file \"{nvdaLogFilePath}\"".format( + baseNVDACommandline=baseNVDACommandline, + nvdaProfileDir=nvdaProfileWorkingDir, + nvdaLogFilePath=nvdaLogFilePath + ), + shell=True, + alias='nvdaAlias', + stdout=_pJoin(testOutputNvdaLogsDir, _createTestIdFileName("stdout.txt")), + stderr=_pJoin(testOutputNvdaLogsDir, _createTestIdFileName("stderr.txt")), + ) + return handle + + def _connectToRemoteServer(self): + """Connects to the nvdaSpyServer + Because we do not know how far through the startup NVDA is, we have to poll + to check that the server is available. Importing the library immediately seems + to succeed, but then calling a keyword later fails with RuntimeError: + "Connection to remote server broken: [Errno 10061] + No connection could be made because the target machine actively refused it" + Instead we wait until the remote server is available before importing the library and continuing. + """ + + builtIn.log("Waiting for nvdaSpy to be available at: {}".format(spyServerURI)) + # Importing the 'Remote' library always succeeds, even when a connection can not be made. + # If that happens, then some 'Remote' keyword will fail at some later point. + # therefore we use '_testRemoteServer' to ensure that we can in fact connect before proceeding. + _blockUntilConditionMet( + getValue=lambda: _testRemoteServer(spyServerURI, log=False), + giveUpAfterSeconds=10, + errorMessage="Unable to connect to nvdaSpy", + ) + builtIn.log("Connecting to nvdaSpy") + maxRemoteKeywordDurationSeconds = 30 # If any remote call takes longer than this, the connection will be closed! + builtIn.import_library( + "Remote", # name of library to import + # Arguments to construct the library instance: + "uri={}".format(spyServerURI), + "timeout={}".format(maxRemoteKeywordDurationSeconds), + # Set an alias for the imported library instance + "WITH NAME", + "nvdaSpy", + ) + builtIn.log("Getting nvdaSpy library instance") + self.nvdaSpy = builtIn.get_library_instance(spyAlias) + self._runNvdaSpyKeyword("set_max_keyword_duration", maxSeconds=maxRemoteKeywordDurationSeconds) + + def _runNvdaSpyKeyword(self, keyword, *args, **kwargs): + if not args: args = [] + if not kwargs: kwargs = {} + builtIn.log("nvdaSpy keyword: {} args: {}, kwargs: {}".format(keyword, args, kwargs)) + return self.nvdaSpy.run_keyword(keyword, args, kwargs) + + def start_NVDA(self, settingsFileName): + self.setup_nvda_profile(settingsFileName) + nvdaProcessHandle = self._startNVDAProcess() + process.process_should_be_running(nvdaProcessHandle) + self._connectToRemoteServer() + self._runNvdaSpyKeyword("wait_for_NVDA_startup_to_complete") + return nvdaProcessHandle + + def save_NVDA_log(self): + """NVDA logs are saved to the ${OUTPUT DIR}/nvdaTestRunLogs/${SUITE NAME}-${TEST NAME}-nvda.log""" + builtIn.log("saving NVDA log") + opSys.create_directory(testOutputNvdaLogsDir) + opSys.copy_file( + nvdaLogFilePath, + _pJoin(testOutputNvdaLogsDir, _createTestIdFileName("nvda.log")) + ) + + def quit_NVDA(self): + builtIn.log("Stopping nvdaSpy server: {}".format(spyServerURI)) + _stopRemoteServer(spyServerURI, log=False) + process.run_process( + "{baseNVDACommandline} -q --disable-addons".format(baseNVDACommandline=baseNVDACommandline), + shell=True, + ) + process.wait_for_process(self.nvdaHandle) + self.save_NVDA_log() + # remove the spy so that if nvda is run manually against this config it does not interfere. + self.teardown_nvda_profile() diff --git a/tests/system/libraries/sendKey.py b/tests/system/libraries/sendKey.py new file mode 100644 index 00000000000..807009abfe5 --- /dev/null +++ b/tests/system/libraries/sendKey.py @@ -0,0 +1,15 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""This file provides system test library functions for sending keyboard key presses. +""" +import pyautogui +pyautogui.FAILSAFE = False + +def send_key(*keys): + """Sends the keys as if pressed by the user. + Full list of keys: pyautogui.KEYBOARD_KEY + """ + pyautogui.hotkey(*keys) diff --git a/tests/system/libraries/speechSpy.py b/tests/system/libraries/speechSpy.py new file mode 100644 index 00000000000..89b29e482f5 --- /dev/null +++ b/tests/system/libraries/speechSpy.py @@ -0,0 +1,28 @@ +#A part of NonVisual Desktop Access (NVDA) +#Copyright (C) 2018 NV Access Limited +#This file is covered by the GNU General Public License. +#See the file COPYING for more details. + +import synthDriverHandler +import extensionPoints + +# inform those who want to know that there is new speech +post_speech = extensionPoints.Action() + +class SynthDriver(synthDriverHandler.SynthDriver): + """A dummy synth driver used by system tests to get speech output + """ + name = "speechSpy" + description = "System test speech spy" + + @classmethod + def check(cls): + return True + + supportedSettings = [] + + def speak(self, speechSequence): + post_speech.notify(speechSequence=speechSequence) + + def cancel(self): + pass diff --git a/tests/system/libraries/systemTestSpy.py b/tests/system/libraries/systemTestSpy.py new file mode 100644 index 00000000000..37af4dfe803 --- /dev/null +++ b/tests/system/libraries/systemTestSpy.py @@ -0,0 +1,210 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""This file provides spy and robot library behaviour for NVDA system tests. +It is copied into the (system test specific) NVDA profile directory. It becomes the '__init__.py' file as part of a +package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library. +""" +import globalPluginHandler +import threading +from systemTestUtils import _blockUntilConditionMet +from logHandler import log +from time import clock as _timer + +import sys +import os +log.debug("before pathmod: {}".format(sys.path)) +# Get the path to the top of the package +TOP_DIR = os.path.abspath(os.path.dirname(__file__)) +# imports that require libraries not distributed with an install of NVDA +sys.path.append( os.path.join(TOP_DIR, "libs")) +log.debug("after pathmod: {}".format(sys.path)) +from robotremoteserver import RobotRemoteServer + +whitespaceMinusSlashN = '\t\x0b\x0c\r ' + + +class SystemTestSpy(object): + SPEECH_HAS_FINISHED_SECONDS = 0.5 + + def __init__(self): + self._nvdaSpeech = [ + [""], # initialise with an empty string, this allows for access via [-1]. This is equiv to no speech. + ] + self._allSpeechStartIndex = 0 + self._speechOccurred = False + self.isNvdaStartupComplete = False + self.lastSpeechTime = _timer() + self._registerWithExtensionPoints() + + def _registerWithExtensionPoints(self): + from core import postNvdaStartup + postNvdaStartup.register(self._onNvdaStartupComplete) + + from synthDrivers.speechSpy import post_speech + post_speech.register(self._onNvdaSpeech) + + # callbacks for extension points + def _onNvdaStartupComplete(self): + self.isNvdaStartupComplete = True + + def _onNvdaSpeech(self, speechSequence=None): + if not speechSequence: return + with threading.Lock(): + self._speechOccurred = True + self.lastSpeechTime = _timer() + self._nvdaSpeech.append(speechSequence) + + # Private helper methods + def _flattenCommandsSeparatingWithNewline(self, commandArray): + f = [c for commands in commandArray for newlineJoined in [commands, [u"\n"]] for c in newlineJoined] + return f + + def _getJoinedBaseStringsFromCommands(self, speechCommandArray): + wsChars = whitespaceMinusSlashN + baseStrings = [c.strip(wsChars) for c in speechCommandArray if isinstance(c, basestring)] + return ''.join(baseStrings).strip() + + # Public methods + def checkIfSpeechOccurredAndReset(self): + # don't let _speechOccurred get updated and overwritten with False + with threading.Lock(): + speechOccurred = self._speechOccurred + self._speechOccurred = False + return speechOccurred + + def getSpeechAtIndex(self, speechIndex): + with threading.Lock(): + return self._getJoinedBaseStringsFromCommands(self._nvdaSpeech[speechIndex]) + + def getSpeechSinceIndex(self, speechIndex): + with threading.Lock(): + speechCommands = self._flattenCommandsSeparatingWithNewline( + self._nvdaSpeech[speechIndex:] + ) + joined = self._getJoinedBaseStringsFromCommands(speechCommands) + return joined + + def getIndexOfLastSpeech(self): + with threading.Lock(): + return len(self._nvdaSpeech) - 1 + + def getIndexOfSpeech(self, speech, startFromIndex=0): + with threading.Lock(): + for index, commands in enumerate(self._nvdaSpeech[startFromIndex:]): + index = index + startFromIndex + baseStrings = [c.strip() for c in commands if isinstance(c, basestring)] + if any(speech in x for x in baseStrings): + return index + return -1 + + def hasSpeechFinished(self): + return self.SPEECH_HAS_FINISHED_SECONDS < _timer() - self.lastSpeechTime + + def dumpSpeechToLog(self): + log.debug("All speech:\n{}".format(repr(self._nvdaSpeech))) + +class NvdaSpyLib(object): + _spy = None # type: SystemTestSpy + + def __init__(self, systemTestSpy): + self._spy = systemTestSpy + self._allSpeechStartIndex = self._spy.getIndexOfLastSpeech() + self._maxKeywordDuration=30 + + def _minTimeout(self, timeout): + """Helper to get the minimum value, the timeout passed in, or self._maxKeywordDuration""" + return min(timeout, self._maxKeywordDuration) + + # Start of Robot library API + + def set_max_keyword_duration(self, maxSeconds): + """This should only be called after importing the library, and should match the 'timeout' value given to the + robot.libraries.Remote instance""" + self._maxKeywordDuration = maxSeconds-1 + + def wait_for_NVDA_startup_to_complete(self): + _blockUntilConditionMet( + getValue=lambda: self._spy.isNvdaStartupComplete, + giveUpAfterSeconds=self._minTimeout(10), + errorMessage="Unable to connect to nvdaSpy", + ) + if self._spy.isNvdaStartupComplete: + self.reset_all_speech_index() + + def get_last_speech(self): + return self._spy.getSpeechAtIndex(-1) + + def get_all_speech(self): + return self._spy.getSpeechSinceIndex(self._allSpeechStartIndex) + + def get_speech_from_index_until_now(self, speechIndex): + return self._spy.getSpeechSinceIndex(speechIndex) + + def reset_all_speech_index(self): + self._allSpeechStartIndex = self._spy.getIndexOfLastSpeech() + return self._allSpeechStartIndex + + def get_last_speech_index(self): + return self._spy.getIndexOfLastSpeech() + + def wait_for_specific_speech(self, speech, sinceIndex=None, maxWaitSeconds=5): + sinceIndex = 0 if not sinceIndex else sinceIndex + success, speechIndex = _blockUntilConditionMet( + getValue=lambda: self._spy.getIndexOfSpeech(speech, sinceIndex), + giveUpAfterSeconds=self._minTimeout(maxWaitSeconds), + shouldStopEvaluator=lambda speechIndex: speechIndex >= 0, + intervalBetweenSeconds=0.1, + errorMessage=None + ) + if not success: + self._spy.dumpSpeechToLog() + raise AssertionError( + "Specific speech did not occur before timeout: {}\n" + "See NVDA log for dump of all speech.".format(speech) + ) + return speechIndex + + def wait_for_speech_to_finish(self, maxWaitSeconds=5.0): + _blockUntilConditionMet( + getValue=self._spy.hasSpeechFinished, + giveUpAfterSeconds=self._minTimeout(maxWaitSeconds), + errorMessage="Speech did not finish before timeout" + ) + +class SystemTestSpyServer(object): + def __init__(self): + self._server = None + + def start(self): + log.debug("TestSpyPlugin started") + spy = SystemTestSpy() # spies on NVDA + server = self._server = RobotRemoteServer( + NvdaSpyLib(spy), # provides library behaviour + port=8270, # default:8270 is `registered by IANA` for remote server usage. Two ASCII values, RF. + serve=False # we want to start this serving on another thread so as not to block. + ) + log.debug("Server address: {}".format(server.server_address)) + server_thread = threading.Thread(target=server.serve) + server_thread.start() + + def stop(self): + log.debug("Stop SystemTestSpyServer called") + self._server.stop() + + +class GlobalPlugin(globalPluginHandler.GlobalPlugin): + + def __init__(self): + super(GlobalPlugin, self).__init__() + self._testSpy = SystemTestSpyServer() + self._testSpy.start() + + def terminate(self): + log.debug("Terminating the systemTestSpy") + self._testSpy.stop() + + __gestures = { + } \ No newline at end of file diff --git a/tests/system/libraries/systemTestUtils.py b/tests/system/libraries/systemTestUtils.py new file mode 100644 index 00000000000..709c47acdc2 --- /dev/null +++ b/tests/system/libraries/systemTestUtils.py @@ -0,0 +1,46 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2018 NV Access Limited +# This file may be used under the terms of the GNU General Public License, version 2 or later. +# For more details see: https://www.gnu.org/licenses/gpl-2.0.html + +"""This file provides utility methods for NVDA system tests. +It is copied into the (system test specific) NVDA profile directory as part of a +package. This allows us to share utility methods between the global plugin and the nvdaRobotLib library. +""" + +from time import sleep as _sleep +from time import clock as _timer + + +def _blockUntilConditionMet( + getValue, + giveUpAfterSeconds, + shouldStopEvaluator=lambda value: bool(value), + intervalBetweenSeconds=0.1, + errorMessage=None): + """Repeatedly tries to get a value up until a time limit expires. Tries are separated by + a time interval. The call will block until shouldStopEvaluator returns True when given the value, + the default evaluator just returns the value converted to a boolean. + @return A tuple, (True, value) if evaluator condition is met, otherwise (False, None) + @raises RuntimeError if the time limit expires and an errorMessage is given. + """ + assert callable(getValue) + assert callable(shouldStopEvaluator) + assert intervalBetweenSeconds > 0.001 + SLEEP_TIME = intervalBetweenSeconds * 0.5 + startTime = _timer() + lastRunTime = startTime + firstRun = True # ensure we start immediately + while (_timer() - startTime) < giveUpAfterSeconds: + if firstRun or (_timer() - lastRunTime) > intervalBetweenSeconds: + firstRun = False + lastRunTime = _timer() + val = getValue() + if shouldStopEvaluator(val): + return True, val + _sleep(SLEEP_TIME) + + else: + if errorMessage: + raise AssertionError(errorMessage) + return False, None diff --git a/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini new file mode 100644 index 00000000000..be26019b045 --- /dev/null +++ b/tests/system/nvdaSettingsFiles/standard-doShowWelcomeDialog.ini @@ -0,0 +1,10 @@ +schemaVersion = 2 +[general] + showWelcomeDialogAtStartup = True +[update] + askedAllowUsageStats = True + autoCheck = False + startupNotification = False + allowUsageStats = False +[speech] + synth = speechSpy diff --git a/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini new file mode 100644 index 00000000000..9cf3c88beda --- /dev/null +++ b/tests/system/nvdaSettingsFiles/standard-dontShowWelcomeDialog.ini @@ -0,0 +1,10 @@ +schemaVersion = 2 +[general] + showWelcomeDialogAtStartup = False +[update] + askedAllowUsageStats = True + autoCheck = False + startupNotification = False + allowUsageStats = False +[speech] + synth = speechSpy diff --git a/tests/system/readme.md b/tests/system/readme.md new file mode 100644 index 00000000000..b85ada68c97 --- /dev/null +++ b/tests/system/readme.md @@ -0,0 +1,79 @@ +## NVDA system tests + +### Dependencies + +The system tests depend on the following: + +- Robot Framework +- Robot Remote Server +- PyAutoGui + +Which can be installed with `pip`: + +``` +pip install robotframework +pip install robotremoteserver +pip install pyautogui +``` + +### Running the tests + +These tests should be run from the windows command prompt (cmd.exe) from the root directory + of your NVDA repository. + + +``` +python -m robot --loglevel DEBUG -d testOutput/system -x systemTests.xml -v whichNVDA:source -P ./tests/system/libraries ./tests/system/ +``` + +The `whichNVDA` argument allows the tests to be run against an installed copy +of NVDA (first ensure it is compatible with the tests). Note valid values are: +* "installed" - when running against the installed version of NVDA, you are likely to get errors in the log unless +the tests are run from an administrator command prompt. +* "source" + +To run a single test or filter tests, use the `--test` argument (wildcards accepted). +Refer to the robot framework documentation for further details. + +``` +python -m robot --test "name of test here" ... +``` + +### Getting the results + +The process is displayed in the command prompt, for more information consider the +`report.html`, `log.html`, and `output.xml` files. The logs from NVDA are saved to the `nvdaTestRunLogs` folder + +### Overview + +Robot Framework loads and parses the test files and their libraries. In our case, generally in the 'setup', +NVDA is started as a new process. It uses a sand box profile, and communication with the test code occurs via an +NVDA addon (`systemTestSpy`). The system test should, as much as possible, interact like a user would. For example, +wait for the speech to confirm that an expected dialog is open before taking the next action to interact. + +Test code goes in robot files, see the robot framework documentation to understand these. +Large strings or other variables go in a variables.py file paired with the robot file. +The `libraries` directory contains files providing "robot keyword" libraries. +The `nvdaSettingsFiles` directory contains various NVDA config files that are used to construct the NVDA +profile in the `%TEMP%` directory. + +### How the test setup works + +This section will not go into the details of robot framework, or robot remote server, +these have their own documentation. There are two major libraries used with the system tests: + +* `nvdaRobotLib` - Provides keywords which help the test code start and cleanup the NVDA process, including the installation of the `systemTestSpy` addon. +* `systemTestSpy` - Is converted into an addon that is installed in the NVDA profile used with the version of NVDA under test. This provides keywords for getting information out of NVDA. For example getting the last speech. + +Helper code in `nvdaRobotLib` is responsible for the construction of the `systemTestSpy` addon. +The addon is constructed as a package from several files: +* `libraries/systemTestSpy.py` becomes `systemTestSpy/__init__.py` +* `libraries/systemTestUtils.py` becomes `systemTestSpy/systemTestUtils.py` +* Files listed in `nvdaRobotLib.requiredPythonImportsForSystemTestSpyPackage` are sourced from locations listed +in the python paths for the instance of python running the robot tests. These are copied to `systemTestSpy/libs/` + +An NVDA profile directory is created in the `%TEMP%` directory, the `systemTestSpy` addon is copied +into the `globalPlugins` directory of this NVDA profile. For each test, an NVDA configuration file +is copied into this profile as well. NVDA is started, and points to this profile directory. At the end of the +test the NVDA log is copied to the robot output directory, under the `nvdaTestRunLogs` directory. The log files are +named by suite and test name. \ No newline at end of file diff --git a/tests/system/sconscript b/tests/system/sconscript new file mode 100644 index 00000000000..c6b800c5fab --- /dev/null +++ b/tests/system/sconscript @@ -0,0 +1,42 @@ +### +#This file is a part of the NVDA project. +#URL: http://www.nvaccess.org/ +#Copyright 2017 NV Access Limited. +#This program is free software: you can redistribute it and/or modify +#it under the terms of the GNU General Public License version 2.0, as published by +#the Free Software Foundation. +#This program is distributed in the hope that it will be useful, +#but WITHOUT ANY WARRANTY; without even the implied warranty of +#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +#This license can be found at: +#http://www.gnu.org/licenses/old-licenses/gpl-2.0.html +### + +import sys +Import("env") + +import os + +tests = env.get("filter") +env = Environment( + ENV={ + 'PATH': os.environ['PATH'], + 'TEMP': os.environ['TEMP'], + 'TMP': os.environ['TMP'], + }) + + +cmd = [ + sys.executable, "-m", "robot", + "--loglevel", "DEBUG", + "-d", "testOutput/system", + "-x", "systemTests.xml", + "-P", "./tests/system/libraries", + "-v", "whichNVDA:source", +] +if tests: + # run specific tests + cmd += ['--test="{}"'.format(tests)] +cmd.append("./tests/system") +target = env.Command(".", None, [cmd]) +Return('target') \ No newline at end of file diff --git a/user_docs/en/changes.t2t b/user_docs/en/changes.t2t index e31d3d559ff..95d6a545e83 100644 --- a/user_docs/en/changes.t2t +++ b/user_docs/en/changes.t2t @@ -28,6 +28,7 @@ What's New in NVDA == Changes for Developers == - Added scriptHandler.script, which can function as a decorator for scripts on scriptable objects. (#6266) +- A system test framework has been introduced for NVDA. (#708) = 2018.2 =