System test with robot framework

.gitignore

@@ -29,6 +29,7 @@ extras/controllerClient/x64
 source/_buildVersion.py
 user_docs/*/keyCommands.t2t
 output
+testOutput
 source/brailleDisplayDrivers/handyTech
 ./developerGuide.html
 user_docs/build.t2tConf

appveyor.yml

@@ -20,6 +20,7 @@ environment:
 
 init:
  - ps: |
+     iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
      if ($env:APPVEYOR_REPO_TAG_NAME -and $env:APPVEYOR_REPO_TAG_NAME.StartsWith("release-")) {
       # Strip "release-" prefix.
       $version = $env:APPVEYOR_REPO_TAG_NAME.Substring(8)
@@ -91,8 +92,8 @@ build_script:
      Set-AppveyorBuildVariable "sconsArgs" $sconsArgs
  - 'echo scons args: %sconsArgs%'
  - py scons.py source %sconsArgs%
- # We don't need launcher to run tests, so run the tests before launcher.
- - py scons.py tests %sconsArgs%
+ # We don't need launcher to run checkPot, so run the checkPot before launcher.
+ - py scons.py checkPot %sconsArgs%
  # The pot gets built by tests, but we don't actually need it as a build artifact.
  - del output\*.pot
  - 'echo scons output targets: %sconsOutTargets%'
@@ -113,6 +114,55 @@ build_script:
  - 7z a -tzip -r ..\output\symbols.zip *.dl_ *.ex_ *.pd_
  - cd ..
 
+before_test:
+ - py -m pip install robotframework robotremoteserver pyautogui nose
+ - mkdir testOutput
+ - mkdir testOutput\unit
+ - mkdir testOutput\system
+ - ps: |
+     $errorCode=0
+     $nvdaLauncherFile=".\output\nvda"
+     if(!$env:release) {
+      $nvdaLauncherFile+="_snapshot"
+     }
+     $nvdaLauncherFile+="_${env:version}.exe"
+     echo NVDALauncherFile: $NVDALauncherFile
+     $outputDir=$(resolve-path .\testOutput)
+     $installerLogFilePath="$outputDir\nvda_install.log"
+     $installerProcess=start-process -FilePath "$nvdaLauncherFile" -ArgumentList "--install-silent --debug-logging --log-file $installerLogFilePath" -passthru
+     try {
+      $installerProcess | wait-process -Timeout 180
+      $errorCode=$installerProcess.ExitCode
+     } catch {
+      echo NVDA installer process timed out
+      $errorCode=1
+     }
+     Push-AppveyorArtifact $installerLogFilePath
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+
+test_script:
+ - ps: |
+     $errorCode=0
+     $outDir = (Resolve-Path .\testOutput\unit\)
+     $unitTestsXml = "$outDir\unitTests.xml"
+     py -m nose --with-xunit --xunit-file="$unitTestsXml" ./tests/unit
+     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
+     Push-AppveyorArtifact $unitTestsXml
+     $wc = New-Object 'System.Net.WebClient'
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", $unitTestsXml)
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+ - ps: |
+     $testOutput = (Resolve-Path .\testOutput\)
+     $systemTestOutput = (Resolve-Path "$testOutput\system")
+     $testSource = "./tests/system"
+     py -m robot --loglevel DEBUG -d $systemTestOutput -x systemTests.xml -v whichNVDA:installed -P "$testSource/libraries" "$testSource"
+     Compress-Archive -Path "$systemTestOutput\*" -DestinationPath "$testOutput\systemTestResult.zip"
+     Push-AppveyorArtifact "$testOutput\systemTestResult.zip"
+     if($LastExitCode -ne 0) { $errorCode=$LastExitCode }
+     $wc = New-Object 'System.Net.WebClient'
+     $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$systemTestOutput\systemTests.xml"))
+     if($errorCode -ne 0) { $host.SetShouldExit($errorCode) }
+
 artifacts:
  - path: output\*
  - path: output\*\*

readme.md

@@ -230,3 +230,16 @@ To run only the translatable string checks (which check that all translatable st
 ```
 scons checkPot
 ```
+
+You may also use scons to run the system tests, though this will still rely on having set up the dependencies (see `tests/system/readme.md`).
+
+```
+scons systemTests
+```
+
+To run only specific system tests, specify them using the `filter` variable on the command line.
+This filter accepts wildcard characters.
+
+```
+scons systemTests filter="Read welcome dialog"
+```

sconstruct

@@ -75,6 +75,8 @@ vars.Add(ListVariable("nvdaHelperDebugFlags", "a list of debugging features you
 vars.Add(EnumVariable('nvdaHelperLogLevel','The level of logging you wish to see, lower is more verbose','15',allowed_values=[str(x) for x in xrange(60)]))
 if "tests" in COMMAND_LINE_TARGETS:
 	vars.Add("unitTests", "A list of unit tests to run", "")
+if "systemTests" in COMMAND_LINE_TARGETS:
+	vars.Add("filter", "A filter for the name of the system test(s) to run. Wildcards accepted.", "")
 
 #Base environment for this and sub sconscripts
 env = Environment(variables=vars,HOST_ARCH='x86',tools=["textfile","gettextTool","t2t",keyCommandsDocTool,'doxygen','recursiveInstall'])

source/core.py

@@ -29,6 +29,11 @@
 from logHandler import log
 import addonHandler
 
+import extensionPoints
+
+# inform those who want to know that NVDA has finished starting up.
+postNvdaStartup = extensionPoints.Action()
+
 PUMP_MAX_DELAY = 10
 
 #: The thread identifier of the main thread.
@@ -470,6 +475,7 @@ def Notify(self):
 		log.debug("initializing updateCheck")
 		updateCheck.initialize()
 	log.info("NVDA initialized")
+	postNvdaStartup.notify()
 
 	log.debug("entering wx application main loop")
 	app.MainLoop()

source/nvda.pyw

@@ -194,6 +194,8 @@ if globalVars.appArgs.debugLogging:
 	logLevel=log.DEBUG
 logHandler.initialize()
 logHandler.log.setLevel(logLevel)
+if logLevel is log.DEBUG:
+	log.debug("Provided arguments: {}".format(sys.argv[1:]))
 
 log.info("Starting NVDA")
 log.debug("Debug level logging enabled")

source/speech.py

@@ -496,7 +496,7 @@ def speak(speechSequence,symbolLevel=None):
 	import speechViewer
 	if speechViewer.isActive:
 		for item in speechSequence:
-			if isinstance(item,basestring):
+			if isinstance(item, basestring):
 				speechViewer.appendText(item)
 	global beenCanceled, curWordChars
 	curWordChars=[]

tests/sconscript

@@ -20,6 +20,11 @@ unitTests = env.SConscript("unit/sconscript", exports=["env"])
 env.Depends(unitTests, sourceDir)
 env.AlwaysBuild(unitTests)
 
+systemTests = env.SConscript("system/sconscript", exports=["env"])
+env.Depends(systemTests, sourceDir)
+env.AlwaysBuild(systemTests)
+env.Alias("systemTests", systemTests)
+
 def checkPotAction(target, source, env):
 	return checkPot.checkPot(source[0].abspath)
 checkPotTarget = env.Command("checkPot", pot, checkPotAction)
@@ -31,10 +36,13 @@ env.Alias("checkPot", checkPotTarget)
 # If specific tests are explicitly specified, only run those.
 explicitUnitTests = env.get("unitTests")
 explicitCheckPot = "checkPot" in COMMAND_LINE_TARGETS
-explicit = explicitUnitTests or explicitCheckPot
+explicitSystemTests = "systemTests" in COMMAND_LINE_TARGETS
+explicit = explicitUnitTests or explicitCheckPot or explicitSystemTests
 tests = []
 if not explicit or explicitUnitTests:
 	tests.append(unitTests)
 if not explicit or explicitCheckPot:
 	tests.append(checkPotTarget)
+if explicit and explicitSystemTests: # only run system tests explicitly
+	tests.append(systemTests)
 env.Alias("tests", tests)

tests/system/.gitignore

@@ -0,0 +1 @@
+nvdaProfile/

tests/system/NVDA Core/startupShutdownNVDA.robot

@@ -0,0 +1,57 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+*** Settings ***
+Documentation	Basic start and exit tests
+Default Tags	NVDA	smoke test
+
+Library	OperatingSystem
+Library	Process
+Library	sendKey.py
+Library	nvdaRobotLib.py
+Library	helperLib.py
+
+Test Setup	start NVDA	standard-dontShowWelcomeDialog.ini
+Test Teardown	quit NVDA
+
+Variables	variables.py
+
+*** Test Cases ***
+Starts
+	[Documentation]	Ensure that NVDA can start
+	process should be running	nvdaAlias
+
+Quits from keyboard
+	[Documentation]	Starts NVDA and ensures that it can be quit using the keyboard
+	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
+
+	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
+	wait for specific speech	${Welcome dialog title}
+	wait for speech to finish
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
+	send key	enter
+
+	${Exit NVDA dialog} =	catenate double space	Exit NVDA	dialog
+	send key	insert	q
+	${INDEX} =	wait for specific speech	${Exit NVDA dialog}
+
+	wait for speech to finish
+	${actual speech} =	get speech from index until now	${INDEX}
+	assert strings are equal	${actual speech}	${QUIT_DIALOG_TEXT}
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
+	send key	enter
+	wait for process	nvdaAlias	timeout=10 sec
+	process should be stopped	nvdaAlias
+
+Read welcome dialog
+	[Documentation]	Ensure that the welcome dialog can be read in full
+	[Setup]	start NVDA	standard-doShowWelcomeDialog.ini
+
+	${Welcome dialog title} =	catenate double space	Welcome to NVDA	dialog
+	${INDEX} =	wait for specific speech	${Welcome dialog title}
+	wait for speech to finish
+	${actual speech} =	get speech from index until now	${INDEX}
+	assert strings are equal	${actual speech}	${WELCOME_DIALOG_TEXT}
+	sleep	1	# the dialog is not always receiving the enter keypress, wait a little longer for it
+	send key	enter

tests/system/NVDA Core/variables.py

@@ -0,0 +1,13 @@
+WELCOME_DIALOG_TEXT = (
+	"Welcome to NVDA  dialog  Welcome to NVDA! Most commands for controlling NVDA require you to hold "
+	"down the NVDA key while pressing other keys. By default, the numpad Insert and main Insert keys "
+	"may both be used as the NVDA key. You can also configure NVDA to use the Caps Lock as the NVDA "
+	"key. Press NVDA plus n at any time to activate the NVDA menu. From this menu, you can configure "
+	"NVDA, get help and access other NVDA functions.\n"
+	"Options  grouping\n"
+	"Keyboard layout:  combo box  desktop  collapsed  Alt plus k"
+)
+QUIT_DIALOG_TEXT = (
+	"Exit NVDA  dialog\n"
+	"What would you like to do?  combo box  Exit  collapsed  Alt plus d"
+)

tests/system/libraries/helperLib.py

@@ -0,0 +1,34 @@
+# A part of NonVisual Desktop Access (NVDA)
+# Copyright (C) 2018 NV Access Limited
+# This file may be used under the terms of the GNU General Public License, version 2 or later.
+# For more details see: https://www.gnu.org/licenses/gpl-2.0.html
+
+"""This file provides general robot library functions for system tests.
+This is in contrast with nvdaRobotLib.py which contains helpers related to starting and stopping NVDA for system
+tests, or with systemTestSpy which contains methods for extracting information about NVDA's behaviour during system
+tests.
+"""
+from robot.libraries.BuiltIn import BuiltIn
+builtIn = BuiltIn()  # type: BuiltIn
+
+def assert_strings_are_equal( actual, expected, ignore_case=False):
+	try:
+		builtIn.should_be_equal_as_strings(
+			actual,
+			expected,
+			msg="Actual speech != Expected speech",
+			ignore_case=ignore_case
+		)
+	except AssertionError:
+		builtIn.log(
+			"repr of actual vs expected (ignore_case={}):\n{}\nvs\n{}".format(
+				ignore_case,
+				repr(actual),
+				repr(expected)
+			)
+		)
+		raise
+
+
+def catenate_double_space(*args):
+	return "  ".join(args)